blob: 91c8a0060c6249443d25f55574ae71a53529dcab (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
#!/usr/bin/env bash
# Downloads the base model (SmolLM2-135M-Instruct) and the dolly-15k dataset
# into the default locations llm-tune expects.
set -euo pipefail
cd "$(dirname "$0")/.."
HF=https://huggingface.co
MODEL=models/smollm2-135m-instruct
DATA=data/dolly-15k
fetch() { # url dest
[ -s "$2" ] && { echo "ok $2"; return; }
echo "get $2"
curl -fL --progress-bar -o "$2.part" "$1" && mv "$2.part" "$2"
}
mkdir -p "$MODEL" "$DATA"
for f in config.json generation_config.json model.safetensors tokenizer.json tokenizer_config.json; do
fetch "$HF/HuggingFaceTB/SmolLM2-135M-Instruct/resolve/main/$f" "$MODEL/$f"
done
fetch "$HF/datasets/databricks/databricks-dolly-15k/resolve/main/databricks-dolly-15k.jsonl" \
"$DATA/databricks-dolly-15k.jsonl"
sha256sum -c --quiet <<SUMS
5af571cbf074e6d21a03528d2330792e532ca608f24ac70a143f6b369968ab8c $MODEL/model.safetensors
2df9083338b4abd6bceb5635764dab5d833b393b55759dffb0959b6fcbf794ec $DATA/databricks-dolly-15k.jsonl
SUMS
echo "done"
|
