hf_download source code

1 /++
2 Download GGUF files from HuggingFace Hub.
3 
4 Without -f, lists .gguf files with sizes.  With -f, downloads with a progress bar.
5 Auth: -t token or HF_TOKEN env var (private repos / higher rate limits).
6 
7 Usage:
8   hf-download -r owner/repo [-f file] [-o outdir] [-t token]
9 +/
10 module hf_download;
11 
12 import std.stdio     : write, writeln, writefln, stderr, stdout;
13 import std.string    : endsWith, indexOf, toStringz;
14 import std.conv      : to;
15 import std.file      : mkdirRecurse, exists, fileWrite = write;
16 import std.path      : buildPath, baseName;
17 import std.net.curl  : HTTP;
18 import std.array     : appender, replicate;
19 import std.exception : enforce;
20 import std.format    : format;
21 import core.stdc.stdio  : fgets, printf, snprintf;
22 import core.stdc.string : strlen;
23 
24 // Listing uses popen+curl; Windows spells these _popen/_pclose.
25 version(Posix)
26 {
27     import core.sys.posix.stdio : popen, pclose;
28 }
29 else version(Windows)
30 {
31     import core.stdc.stdio : FILE;
32     extern(C) nothrow @nogc
33     {
34         FILE* _popen(scope const char* cmd, scope const char* mode);
35         int   _pclose(FILE* stream);
36     }
37     alias popen  = _popen;
38     alias pclose = _pclose;
39 }
40 
41 
42 version(Posix)
43     import core.sys.posix.stdlib : c_getenv = getenv;
44 else version(Windows)
45     extern(C) char* getenv(scope const char*) nothrow @nogc;
46 
47 enum HF_API  = "https://huggingface.co/api";
48 enum HF_BASE = "https://huggingface.co";
49 
50 struct GgufFile
51 {
52     string name;
53     ulong  size; // bytes; 0 if unavailable
54 }
55 
56 int main(string[] args)
57 {
58     struct Options
59     {
60         string repo;
61         string filename;
62         string outDir  = ".";
63         string token;
64         bool   listAll;
65         bool   help;
66     }
67 
68     Options opts;
69     opts.token = envGet("HF_TOKEN");
70 
71     for (int i = 1; i < cast(int) args.length; i++)
72         with (opts) switch (args[i])
73         {
74             case "-r": if (++i < cast(int) args.length) repo     = args[i]; else return printUsage(args[0]); break;
75             case "-f": if (++i < cast(int) args.length) filename = args[i]; else return printUsage(args[0]); break;
76             case "-o": if (++i < cast(int) args.length) outDir   = args[i]; else return printUsage(args[0]); break;
77             case "-t": if (++i < cast(int) args.length) token    = args[i]; else return printUsage(args[0]); break;
78             case "-l", "--list": listAll = true; break;
79             case "-h", "--help": help    = true; break;
80             default:
81                 stderr.writefln("unknown option: %s", args[i]);
82                 return printUsage(args[0]);
83         }
84 
85     if (args.length < 2 || opts.help)
86         return printUsage(args[0]);
87 
88     if (opts.repo.length == 0)
89     {
90         stderr.writeln("error: -r owner/repo is required");
91         return printUsage(args[0]);
92     }
93 
94     if (opts.filename.length == 0 || opts.listAll)
95         return listGgufFiles(opts.repo, opts.token);
96     else
97         return downloadFile(opts.repo, opts.filename, opts.outDir, opts.token);
98 }
99 
100 // ── List ──────────────────────────────────────────────────────────────────────
101 
102 int listGgufFiles(string repo, string token)
103 {
104     // ?blobs=true includes LFS size for each sibling.
105     immutable url = HF_API ~ "/models/" ~ repo ~ "?blobs=true";
106 
107     string body_;
108     try
109         body_ = httpGet(url, token);
110     catch (Exception e)
111     {
112         stderr.writefln("error: %s", e.msg);
113         return 1;
114     }
115 
116     if (hasJsonKey(body_, "error"))
117     {
118         stderr.writefln("API error: %s", extractJsonString(body_, "error"));
119         return 1;
120     }
121     if (!hasJsonKey(body_, "siblings"))
122     {
123         stderr.writeln("error: unexpected API response");
124         return 1;
125     }
126 
127     auto files = extractGgufFiles(body_);
128     insertionSort(files);
129 
130     if (files.length == 0)
131     {
132         writefln("No .gguf files found in %s", repo);
133         return 0;
134     }
135 
136     int maxLen = 0;
137     foreach (ref f; files)
138         if (cast(int) f.name.length > maxLen)
139             maxLen = cast(int) f.name.length;
140 
141     writefln("GGUF files in %s  (%d found)", repo, files.length);
142     printFileList(files, maxLen);
143     writeln();
144     writefln("Download: hf-download -r %s -f <filename>", repo);
145     return 0;
146 }
147 
148 // ── Download ──────────────────────────────────────────────────────────────────
149 
150 int downloadFile(string repo, string filename, string outDir, string token)
151 {
152     immutable url     = HF_BASE ~ "/" ~ repo ~ "/resolve/main/" ~ filename;
153     immutable outPath = buildPath(outDir, baseName(filename));
154 
155     if (outDir != "." && !exists(outDir))
156         mkdirRecurse(outDir);
157 
158     writefln("Downloading: %s", filename);
159     writefln("  from : %s", url);
160     writefln("  to   : %s", outPath);
161     writeln();
162 
163     try
164         download(url, outPath, token);
165     catch (Exception e)
166     {
167         stderr.writefln("error: %s", e.msg);
168         return 1;
169     }
170 
171     writefln("Saved: %s", outPath);
172     return 0;
173 }
174 
175 void download(string url, string fileName, string token = "") @trusted
176 {
177     auto buf = appender!(ubyte[])();
178     size_t contentLength;
179     auto http = HTTP(url);
180     if (token.length)
181         http.addRequestHeader("Authorization", "Bearer " ~ token);
182     http.onReceiveHeader((in k, in v) {
183         if (k == "content-length")
184             contentLength = to!size_t(v);
185     });
186 
187     enum barWidth = 50;
188     http.onReceive((data) {
189         buf.put(data);
190         if (contentLength)
191         {
192             float progress = cast(float) buf.data.length / contentLength;
193             write("\r[",
194                 "=".replicate(cast(int)(barWidth * progress)), ">",
195                 " ".replicate(barWidth - cast(int)(barWidth * progress)),
196                 "] ", format("%d%%", cast(int)(progress * 100)));
197             stdout.flush();
198         }
199         return data.length;
200     });
201 
202     http.perform();
203     enforce(http.statusLine.code / 100 == 2 || http.statusLine.code == 302,
204         format("HTTP request failed: %s", http.statusLine.code));
205     fileWrite(fileName, buf.data);
206     writeln();
207 }
208 
209 // ── HTTP ──────────────────────────────────────────────────────────────────────
210 
211 string httpGet(string url, string token) @trusted
212 {
213     string cmd = "curl -sf -L";
214     if (token.length)
215         cmd ~= " -H \"Authorization: Bearer " ~ token ~ "\"";
216     cmd ~= " \"" ~ url ~ "\"";
217 
218     auto pipe = popen(cmd.toStringz, "r");
219     if (pipe is null)
220         throw new Exception("popen failed");
221 
222     string result;
223     char[4096] tmp = void;
224     while (fgets(tmp.ptr, cast(int) tmp.sizeof, pipe) !is null)
225         result ~= tmp.ptr[0 .. strlen(tmp.ptr)];
226 
227     int rc = pclose(pipe);
228     if (rc != 0)
229         throw new Exception("curl exited with code " ~ rc.to!string);
230 
231     return result;
232 }
233 
234 // ── JSON helpers ──────────────────────────────────────────────────────────────
235 
236 bool hasJsonKey(string json, string key) pure nothrow @safe
237 {
238     return json.indexOf("\"" ~ key ~ "\"") >= 0;
239 }
240 
241 string extractJsonString(string json, string key) pure @safe
242 {
243     auto needle = "\"" ~ key ~ "\"";
244     auto kpos   = json.indexOf(needle);
245     if (kpos < 0) return "";
246     auto pos = kpos + needle.length;
247     while (pos < json.length && (json[pos] == ' ' || json[pos] == ':')) pos++;
248     if (pos >= json.length || json[pos] != '"') return "";
249     pos++;
250     auto end = json.indexOf('"', pos);
251     if (end < 0) return "";
252     return json[pos .. end];
253 }
254 
255 // Parses name + byte size from each sibling in a ?blobs=true API response.
256 GgufFile[] extractGgufFiles(string json) pure @safe
257 {
258     GgufFile[] files;
259     enum rfnKey = "\"rfilename\"";
260     enum szKey  = "\"size\"";
261     ptrdiff_t pos = 0;
262     while (true)
263     {
264         auto kpos = json.indexOf(rfnKey, pos);
265         if (kpos < 0) break;
266         pos = kpos + rfnKey.length;
267 
268         while (pos < json.length &&
269                (json[pos] == ' ' || json[pos] == ':' || json[pos] == '\t'))
270             pos++;
271         if (pos >= json.length || json[pos] != '"') continue;
272         pos++;
273         auto end = json.indexOf('"', pos);
274         if (end < 0) break;
275         auto name = json[pos .. end];
276         pos = end + 1;
277 
278         if (!name.endsWith(".gguf")) continue;
279 
280         // Search for "size" only within this sibling's object, not the next.
281         auto nextRfn  = json.indexOf(rfnKey, pos);
282         auto searchTo = nextRfn < 0 ? cast(ptrdiff_t) json.length : nextRfn;
283 
284         ulong sz  = 0;
285         auto  szp = json.indexOf(szKey, pos);
286         if (szp >= 0 && szp < searchTo)
287         {
288             auto p = szp + szKey.length;
289             while (p < json.length &&
290                    (json[p] == ' ' || json[p] == ':' || json[p] == '\t'))
291                 p++;
292             ulong n = 0;
293             bool  ok = false;
294             while (p < json.length && json[p] >= '0' && json[p] <= '9')
295             {
296                 n  = n * 10 + (json[p] - '0');
297                 p++;
298                 ok = true;
299             }
300             if (ok) sz = n;
301         }
302 
303         files ~= GgufFile(name, sz);
304     }
305     return files;
306 }
307 
308 void insertionSort(GgufFile[] arr) @safe nothrow
309 {
310     for (size_t i = 1; i < arr.length; i++)
311     {
312         GgufFile key = arr[i];
313         ptrdiff_t j  = cast(ptrdiff_t) i - 1;
314         while (j >= 0 && arr[j].name > key.name)
315         {
316             arr[j + 1] = arr[j];
317             j--;
318         }
319         arr[j + 1] = key;
320     }
321 }
322 
323 void printFileList(GgufFile[] files, int maxLen) @trusted nothrow
324 {
325     enum sep = "─";
326     for (int i = 0; i < maxLen + 12; i++) printf("%s", sep.ptr);
327     printf("\n");
328 
329     ulong total = 0;
330     foreach (ref f; files)
331     {
332         total += f.size;
333         if (f.size > 0)
334             printf("  %-*s  %s\n", maxLen, f.name.toStringz, fmtBytes(f.size));
335         else
336             printf("  %s\n", f.name.toStringz);
337     }
338 
339     if (total > 0)
340         printf("  %*s  %s\n", maxLen, "Total".ptr, fmtBytes(total));
341 }
342 
343 // ── Env ───────────────────────────────────────────────────────────────────────
344 
345 string envGet(string name) @trusted nothrow
346 {
347     version(Posix)
348         auto p = c_getenv(name.toStringz);
349     else
350         auto p = getenv(name.toStringz);
351     if (p is null) return "";
352     return cast(string) p[0 .. strlen(p)];
353 }
354 
355 // ── Formatting ────────────────────────────────────────────────────────────────
356 const(char)* fmtBytes(ulong n) @trusted nothrow
357 {
358     static char[32][2] bufs;
359     static int which = 0;
360     which = 1 - which;
361     char* b = bufs[which].ptr;
362 
363     double gb = cast(double) n / (1024.0 * 1024 * 1024);
364     double mb = cast(double) n / (1024.0 * 1024);
365     double kb = cast(double) n / 1024.0;
366 
367     if (n >= 1024UL * 1024 * 1024)
368         snprintf(b, 32, "%.2f GiB", gb);
369     else if (n >= 1024UL * 1024)
370         snprintf(b, 32, "%.2f MiB", mb);
371     else if (n >= 1024)
372         snprintf(b, 32, "%.2f KiB", kb);
373     else
374         snprintf(b, 32, "%llu B", cast(ulong) n);
375 
376     return b;
377 }
378 
379 string humanBytes(ulong n) @trusted nothrow
380 {
381     auto p = fmtBytes(n);
382     return cast(string) p[0 .. strlen(p)];
383 }
384 
385 // ── Usage ─────────────────────────────────────────────────────────────────────
386 
387 int printUsage(string prog) @trusted nothrow
388 {
389     auto p = prog.toStringz;
390     printf(
391         "\nusage: %s -r owner/repo [-f file] [-o outdir] [-t token]\n\n"
392         ~ "  -r   HuggingFace repository  (e.g. unsloth/Qwen3-0.6B-GGUF)\n"
393         ~ "  -f   filename to download    (omit to list .gguf files)\n"
394         ~ "  -o   output directory        (default: current directory)\n"
395         ~ "  -t   HF access token         (or set HF_TOKEN env var)\n"
396         ~ "  -l   list .gguf files and exit\n"
397         ~ "  -h   show this help\n\n"
398         ~ "examples:\n"
399         ~ "  %s -r unsloth/Qwen3-0.6B-GGUF\n"
400         ~ "  %s -r unsloth/Qwen3-0.6B-GGUF -f Qwen3-0.6B-Q4_K_M.gguf -o ~/models\n"
401         ~ "  HF_TOKEN=hf_xxx %s -r org/private-model -f model.gguf\n\n",
402         p, p, p, p);
403     return 1;
404 }