nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 23 Packages Projects Releases Wiki Activity

common : update download code (#19573)

Browse Source 
* common : remove legacy .json to .etag migration code

Signed-off-by: Adrien Gallouët <angt@huggingface.co>

* common : simplify common_download_file_single_online

This commit also force a redownload if the file exists
but has no .etag file.

Signed-off-by: Adrien Gallouët <angt@huggingface.co>

---------

Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët
2026-02-13 15:10:46 +01:00
committed by GitHub
parent 752584d5f5
commit b48e80f677
1 changed files with 74 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/download.cpp
+35 -68
View File
@@ -114,44 +114,18 @@ static void write_etag(const std::string & path, const std::string & etag) {
} }
static std::string read_etag(const std::string & path) { static std::string read_etag(const std::string & path) {
std::string none;
const std::string etag_path = path + ".etag"; const std::string etag_path = path + ".etag";
if (!std::filesystem::exists(etag_path)) {
if (std::filesystem::exists(etag_path)) { return {};
}
std::ifstream etag_in(etag_path); std::ifstream etag_in(etag_path);
if (!etag_in) { if (!etag_in) {
LOG_ERR("%s: could not open .etag file for reading: %s\n", __func__, etag_path.c_str()); LOG_ERR("%s: could not open .etag file for reading: %s\n", __func__, etag_path.c_str());
return none; return {};
} }
std::string etag; std::string etag;
std::getline(etag_in, etag); std::getline(etag_in, etag);
return etag; return etag;
}
// no etag file, but maybe there is an old .json
// remove this code later
const std::string metadata_path = path + ".json";
if (std::filesystem::exists(metadata_path)) {
std::ifstream metadata_in(metadata_path);
try {
nlohmann::json metadata_json;
metadata_in >> metadata_json;
LOG_DBG("%s: previous metadata file found %s: %s\n", __func__, metadata_path.c_str(),
metadata_json.dump().c_str());
if (metadata_json.contains("etag") && metadata_json.at("etag").is_string()) {
std::string etag = metadata_json.at("etag");
write_etag(path, etag);
if (!std::filesystem::remove(metadata_path)) {
LOG_WRN("%s: failed to delete old .json metadata file: %s\n", __func__, metadata_path.c_str());
}
return etag;
}
} catch (const nlohmann::json::exception & e) {
LOG_ERR("%s: error reading metadata file %s: %s\n", __func__, metadata_path.c_str(), e.what());
}
}
return none;
} }
static bool is_http_status_ok(int status) { static bool is_http_status_ok(int status) {
@@ -347,51 +321,44 @@ static int common_download_file_single_online(const std::string & url,
LOG_INF("%s: no previous model file found %s\n", __func__, path.c_str()); LOG_INF("%s: no previous model file found %s\n", __func__, path.c_str());
} }
for (int i = 0; i < max_attempts; ++i) {
auto head = cli.Head(parts.path); auto head = cli.Head(parts.path);
bool head_ok = head && head->status >= 200 && head->status < 300; if (!head || head->status < 200 || head->status >= 300) {
if (!head_ok) { LOG_WRN("%s: HEAD failed, status: %d\n", __func__, head ? head->status : -1);
LOG_WRN("%s: HEAD invalid http status code received: %d\n", __func__, head ? head->status : -1);
if (file_exists) { if (file_exists) {
LOG_INF("%s: Using cached file (HEAD failed): %s\n", __func__, path.c_str()); LOG_INF("%s: using cached file (HEAD failed): %s\n", __func__, path.c_str());
return 304; // 304 Not Modified - fake cached response return 304; // 304 Not Modified - fake cached response
} }
return head->status; // cannot use cached file, return raw status code return head ? head->status : -1;
// TODO: maybe retry only on certain codes
} }
std::string etag; std::string etag;
if (head_ok && head->has_header("ETag")) { if (head->has_header("ETag")) {
etag = head->get_header_value("ETag"); etag = head->get_header_value("ETag");
} }
size_t total_size = 0; size_t total_size = 0;
if (head_ok && head->has_header("Content-Length")) { if (head->has_header("Content-Length")) {
try { try {
total_size = std::stoull(head->get_header_value("Content-Length")); total_size = std::stoull(head->get_header_value("Content-Length"));
} catch (const std::exception& e) { } catch (const std::exception& e) {
LOG_WRN("%s: Invalid Content-Length in HEAD response: %s\n", __func__, e.what()); LOG_WRN("%s: invalid Content-Length in HEAD response: %s\n", __func__, e.what());
} }
} }
bool supports_ranges = false; bool supports_ranges = false;
if (head_ok && head->has_header("Accept-Ranges")) { if (head->has_header("Accept-Ranges")) {
supports_ranges = head->get_header_value("Accept-Ranges") != "none"; supports_ranges = head->get_header_value("Accept-Ranges") != "none";
} }
bool should_download_from_scratch = false;
if (!last_etag.empty() && !etag.empty() && last_etag != etag) {
LOG_WRN("%s: ETag header is different (%s != %s): triggering a new download\n", __func__,
last_etag.c_str(), etag.c_str());
should_download_from_scratch = true;
}
if (file_exists) { if (file_exists) {
if (!should_download_from_scratch) { if (etag.empty()) {
LOG_INF("%s: using cached file: %s\n", __func__, path.c_str()); LOG_INF("%s: using cached file (no server etag): %s\n", __func__, path.c_str());
return 304; // 304 Not Modified - fake cached response
}
if (!last_etag.empty() && last_etag == etag) {
LOG_INF("%s: using cached file (same etag): %s\n", __func__, path.c_str());
return 304; // 304 Not Modified - fake cached response return 304; // 304 Not Modified - fake cached response
} }
LOG_WRN("%s: deleting previous downloaded file: %s\n", __func__, path.c_str());
if (remove(path.c_str()) != 0) { if (remove(path.c_str()) != 0) {
LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str()); LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str());
return -1; return -1;
@@ -399,10 +366,19 @@ static int common_download_file_single_online(const std::string & url,
} }
const std::string path_temporary = path + ".downloadInProgress"; const std::string path_temporary = path + ".downloadInProgress";
int delay = retry_delay_seconds;
for (int i = 0; i < max_attempts; ++i) {
if (i) {
LOG_WRN("%s: retrying after %d seconds...\n", __func__, delay);
std::this_thread::sleep_for(std::chrono::seconds(delay));
delay *= retry_delay_seconds;
}
size_t existing_size = 0; size_t existing_size = 0;
if (std::filesystem::exists(path_temporary)) { if (std::filesystem::exists(path_temporary)) {
if (supports_ranges && !should_download_from_scratch) { if (supports_ranges) {
existing_size = std::filesystem::file_size(path_temporary); existing_size = std::filesystem::file_size(path_temporary);
} else if (remove(path_temporary.c_str()) != 0) { } else if (remove(path_temporary.c_str()) != 0) {
LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str()); LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str());
@@ -410,21 +386,11 @@ static int common_download_file_single_online(const std::string & url,
} }
} }
// start the download LOG_INF("%s: downloading from %s to %s (etag:%s)...\n",
LOG_INF("%s: trying to download model from %s to %s (etag:%s)...\n", __func__, common_http_show_masked_url(parts).c_str(),
__func__, common_http_show_masked_url(parts).c_str(), path_temporary.c_str(), etag.c_str()); path_temporary.c_str(), etag.c_str());
const bool was_pull_successful = common_pull_file(cli, parts.path, path_temporary, supports_ranges, existing_size, total_size);
if (!was_pull_successful) {
if (i + 1 < max_attempts) {
const int exponential_backoff_delay = std::pow(retry_delay_seconds, i) * 1000;
LOG_WRN("%s: retrying after %d milliseconds...\n", __func__, exponential_backoff_delay);
std::this_thread::sleep_for(std::chrono::milliseconds(exponential_backoff_delay));
} else {
LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
}
continue;
}
if (common_pull_file(cli, parts.path, path_temporary, supports_ranges, existing_size, total_size)) {
if (std::rename(path_temporary.c_str(), path.c_str()) != 0) { if (std::rename(path_temporary.c_str(), path.c_str()) != 0) {
LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str()); LOG_ERR("%s: unable to rename file: %s to %s\n", __func__, path_temporary.c_str(), path.c_str());
return -1; return -1;
@@ -432,10 +398,11 @@ static int common_download_file_single_online(const std::string & url,
if (!etag.empty()) { if (!etag.empty()) {
write_etag(path, etag); write_etag(path, etag);
} }
return head->status;
return head->status; // TODO: use actual GET status? }
} }
LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts);
return -1; // max attempts reached return -1; // max attempts reached
} }