server: clean up static assets handling (#24550)
* server: clean up static assets handling * nits * simplify file name handling, use static file name everywhere * cmake/ui : bundle UI assets in an archive * ui : run prettier on post-build.js --------- Co-authored-by: Alde Rojas <hello@alde.dev>
This commit is contained in:
Xuan-Son Nguyen
co-authored by
Alde Rojas
Alde Rojas
parent
d8a24ccee2
commit
57fe1f07c3
@@ -1,6 +1,6 @@
|
||||
set(TARGET llama-ui)
|
||||
|
||||
set(LLAMA_UI_HF_BUCKET "llama-ui" CACHE STRING "Hugging Face bucket name for prebuilt UI assets")
|
||||
set(LLAMA_UI_HF_BUCKET "ggml-org/llama-ui" CACHE STRING "Hugging Face bucket name for prebuilt UI assets")
|
||||
|
||||
# Backward compat: forward old var to new one
|
||||
if(DEFINED LLAMA_BUILD_WEBUI)
|
||||
|
||||
+105
-25
@@ -1,16 +1,42 @@
|
||||
// llama-ui-embed: generate ui.cpp / ui.h that embed UI assets as C arrays.
|
||||
//
|
||||
// Usage:
|
||||
// llama-ui-embed <out_cpp> <out_h> [<asset_name> <asset_path>]...
|
||||
// llama-ui-embed <out_cpp> <out_h> [<asset_dir>]
|
||||
//
|
||||
// Embeds every regular file directly under <asset_dir> (non-recursive).
|
||||
// Without <asset_dir>, emits an empty asset table.
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <cinttypes>
|
||||
#include <cstdint>
|
||||
|
||||
|
||||
static const char * mime_from_ext(const std::string & name) {
|
||||
auto ext = name.rfind('.');
|
||||
if (ext == std::string::npos) return "application/octet-stream";
|
||||
std::string e = name.substr(ext + 1);
|
||||
if (e == "html") return "text/html; charset=utf-8";
|
||||
if (e == "css") return "text/css";
|
||||
if (e == "js") return "application/javascript";
|
||||
if (e == "json") return "application/json";
|
||||
if (e == "webmanifest") return "application/manifest+json";
|
||||
if (e == "svg") return "image/svg+xml";
|
||||
if (e == "png") return "image/png";
|
||||
if (e == "jpg" ||
|
||||
e == "jpeg") return "image/jpeg";
|
||||
if (e == "ico") return "image/x-icon";
|
||||
if (e == "woff") return "font/woff";
|
||||
if (e == "woff2") return "font/woff2";
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
// Computes FNV-1a hash of the data
|
||||
static uint64_t fnv_hash(const uint8_t * data, size_t len) {
|
||||
@@ -24,10 +50,10 @@ static uint64_t fnv_hash(const uint8_t * data, size_t len) {
|
||||
return hash;
|
||||
}
|
||||
|
||||
static bool read_file(const std::string & path, std::vector<unsigned char> & out) {
|
||||
static bool read_file(const std::filesystem::path & path, std::vector<unsigned char> & out) {
|
||||
std::ifstream f(path, std::ios::binary | std::ios::ate);
|
||||
if (!f) {
|
||||
fprintf(stderr, "embed: cannot open %s\n", path.c_str());
|
||||
fprintf(stderr, "embed: cannot open %s\n", path.string().c_str());
|
||||
return false;
|
||||
}
|
||||
const auto sz = f.tellg();
|
||||
@@ -89,70 +115,124 @@ static std::string fmt(const char * pattern, ...) {
|
||||
return (n > 0) ? std::string(tmp, static_cast<size_t>(n)) : std::string();
|
||||
}
|
||||
|
||||
struct asset_entry {
|
||||
std::string name;
|
||||
std::filesystem::path path;
|
||||
};
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc < 3 || ((argc - 3) % 2) != 0) {
|
||||
fprintf(stderr, "usage: %s <out_cpp> <out_h> [<name> <path>]...\n", argv[0]);
|
||||
if (argc < 3 || argc > 4) {
|
||||
fprintf(stderr, "usage: %s <out_cpp> <out_h> [<asset_dir>]\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string out_cpp = argv[1];
|
||||
const std::string out_h = argv[2];
|
||||
const int n_assets = (argc - 3) / 2;
|
||||
|
||||
std::vector<asset_entry> assets;
|
||||
if (argc == 4) {
|
||||
const std::filesystem::path dir = argv[3];
|
||||
|
||||
std::error_code ec;
|
||||
std::filesystem::directory_iterator it(dir, ec);
|
||||
if (ec) {
|
||||
fprintf(stderr, "embed: cannot iterate %s: %s\n", argv[3], ec.message().c_str());
|
||||
return 1;
|
||||
}
|
||||
for (const auto & entry : it) {
|
||||
if (!entry.is_regular_file()) {
|
||||
continue;
|
||||
}
|
||||
assets.push_back({ entry.path().filename().generic_string(), entry.path() });
|
||||
}
|
||||
|
||||
// directory iteration order is unspecified; sort for reproducible output
|
||||
std::sort(assets.begin(), assets.end(),
|
||||
[](const asset_entry & a, const asset_entry & b) { return a.name < b.name; });
|
||||
}
|
||||
|
||||
const int n_assets = static_cast<int>(assets.size());
|
||||
|
||||
if (n_assets > 0) {
|
||||
bool has_index = false, has_bundle_js = false, has_bundle_css = false, has_version = false;
|
||||
for (const auto & a : assets) {
|
||||
if (a.name == "index.html") has_index = true;
|
||||
if (a.name == "bundle.js") has_bundle_js = true;
|
||||
if (a.name == "bundle.css") has_bundle_css = true;
|
||||
if (a.name == "version.json") has_version = true;
|
||||
}
|
||||
if (!has_index || !has_bundle_js || !has_bundle_css || !has_version) {
|
||||
fprintf(stderr, "embed: missing required assets (need index.html, bundle.js, bundle.css, version.json); got:\n");
|
||||
for (const auto & a : assets) {
|
||||
fprintf(stderr, " %s\n", a.name.c_str());
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
std::string h;
|
||||
h += "#pragma once\n\n#include <stddef.h>\n\n";
|
||||
h += "#pragma once\n\n#include <array>\n#include <string>\n\n";
|
||||
if (n_assets > 0) {
|
||||
h += "#define LLAMA_UI_HAS_ASSETS 1\n\n";
|
||||
}
|
||||
h +=
|
||||
"struct llama_ui_asset {\n"
|
||||
" const char * name;\n"
|
||||
" std::string name;\n"
|
||||
" const unsigned char * data;\n"
|
||||
" size_t size;\n"
|
||||
" const char * etag;\n"
|
||||
" std::size_t size;\n"
|
||||
" std::string etag;\n"
|
||||
" std::string type;\n"
|
||||
"};\n\n"
|
||||
"const llama_ui_asset * llama_ui_find_asset(const char * name);\n";
|
||||
"const llama_ui_asset * llama_ui_find_asset(const std::string & name);\n";
|
||||
h += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets();\n", n_assets);
|
||||
|
||||
std::string cpp;
|
||||
cpp += "#include \"ui.h\"\n\n#include <string.h>\n\n";
|
||||
cpp += "#include \"ui.h\"\n\n";
|
||||
|
||||
if (n_assets > 0) {
|
||||
for (int i = 0; i < n_assets; i++) {
|
||||
const char * path = argv[3 + i * 2 + 1];
|
||||
std::vector<unsigned char> bytes;
|
||||
if (!read_file(path, bytes)) {
|
||||
if (!read_file(assets[i].path, bytes)) {
|
||||
return 1;
|
||||
}
|
||||
cpp += fmt("static const unsigned char asset_%d_data[] = {", i);
|
||||
append_bytes_hex(cpp, bytes);
|
||||
const auto hash = fnv_hash(bytes.data(), bytes.size());
|
||||
|
||||
cpp += fmt("};\nstatic const size_t asset_%d_size = %zu;\n",
|
||||
cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n",
|
||||
i, bytes.size());
|
||||
cpp += fmt("static const char asset_%d_etag[] = \"\\\"0x%016" PRIx64 "\\\"\";\n\n",
|
||||
cpp += fmt("static const char asset_%d_etag[] = \"\\\"0x%016" PRIx64 "\\\"\";\n\n",
|
||||
i, hash);
|
||||
}
|
||||
|
||||
cpp += "static const llama_ui_asset g_assets[] = {\n";
|
||||
cpp += fmt("static const std::array<llama_ui_asset, %d> g_assets = {{\n", n_assets);
|
||||
for (int i = 0; i < n_assets; i++) {
|
||||
cpp += fmt(" { \"%s\", asset_%d_data, asset_%d_size, asset_%d_etag },\n",
|
||||
argv[3 + i * 2], i, i, i);
|
||||
const std::string & name = assets[i].name;
|
||||
cpp += fmt(" { \"%s\", asset_%d_data, asset_%d_size, asset_%d_etag, \"%s\" },\n",
|
||||
name.c_str(), i, i, i, mime_from_ext(name));
|
||||
}
|
||||
cpp += "};\n\n";
|
||||
cpp += "}};\n\n";
|
||||
|
||||
cpp +=
|
||||
"const llama_ui_asset * llama_ui_find_asset(const char * name) {\n"
|
||||
"const llama_ui_asset * llama_ui_find_asset(const std::string & name) {\n"
|
||||
" for (const auto & a : g_assets) {\n"
|
||||
" if (strcmp(a.name, name) == 0) {\n"
|
||||
" if (a.name == name) {\n"
|
||||
" return &a;\n"
|
||||
" }\n"
|
||||
" }\n"
|
||||
" return nullptr;\n"
|
||||
"}\n";
|
||||
cpp += fmt("const std::array<llama_ui_asset, %d> & llama_ui_get_assets() {\n", n_assets);
|
||||
cpp += " return g_assets;\n"
|
||||
"}\n";
|
||||
} else {
|
||||
cpp +=
|
||||
"const llama_ui_asset * llama_ui_find_asset(const char *) {\n"
|
||||
"const llama_ui_asset * llama_ui_find_asset(const std::string &) {\n"
|
||||
" return nullptr;\n"
|
||||
"}\n"
|
||||
"const std::array<llama_ui_asset, 0> & llama_ui_get_assets() {\n"
|
||||
" static const std::array<llama_ui_asset, 0> empty{};\n"
|
||||
" return empty;\n"
|
||||
"}\n";
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "npm run build-pwa-assets && vite build",
|
||||
"build": "npm run build-pwa-assets && vite build && node scripts/post-build.js",
|
||||
"build-pwa-assets": "npx @vite-pwa/assets-generator --root . --config pwa-assets.config.ts && npx @vite-pwa/assets-generator --root . --config pwa-assets-dark.config.ts && node scripts/make-icons-circular.js",
|
||||
"dev": "bash scripts/dev.sh",
|
||||
"preview": "vite preview",
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env node
|
||||
// Post-build: copy hashed/nested assets to predictable flat names.
|
||||
// No file content is modified — the C++ server handles routing hashed URLs
|
||||
// to the correct stored asset at runtime.
|
||||
//
|
||||
// Copies:
|
||||
// _app/immutable/bundle.HASH.js -> bundle.js
|
||||
// _app/immutable/assets/bundle.HASH.css -> bundle.css
|
||||
// workbox-HEXHASH.js -> workbox.js
|
||||
// _app/version.json -> version.json
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const outDir = process.env.LLAMA_UI_OUT_DIR ?? './dist';
|
||||
|
||||
function findOne(dir, pattern) {
|
||||
const files = fs.readdirSync(dir).filter((f) => pattern.test(f));
|
||||
if (files.length === 0) throw new Error(`post-build: no file matching ${pattern} in ${dir}`);
|
||||
return path.join(dir, files[0]);
|
||||
}
|
||||
|
||||
function copyFlat(src, destName) {
|
||||
const dest = path.join(outDir, destName);
|
||||
fs.copyFileSync(src, dest);
|
||||
console.log(`post-build: ${path.relative(outDir, src)} -> ${destName}`);
|
||||
}
|
||||
|
||||
const bundleJs = findOne(path.join(outDir, '_app/immutable'), /^bundle\.[^.]+\.js$/);
|
||||
const bundleCss = findOne(path.join(outDir, '_app/immutable/assets'), /^bundle\.[^.]+\.css$/);
|
||||
const workbox = findOne(outDir, /^workbox-[0-9a-f]+\.js$/);
|
||||
|
||||
copyFlat(bundleJs, 'bundle.js');
|
||||
copyFlat(bundleCss, 'bundle.css');
|
||||
copyFlat(workbox, 'workbox.js');
|
||||
|
||||
const versionSrc = path.join(outDir, '_app/version.json');
|
||||
if (fs.existsSync(versionSrc)) {
|
||||
copyFlat(versionSrc, 'version.json');
|
||||
}
|
||||
@@ -23,8 +23,7 @@ export function buildInfoPlugin(): Plugin {
|
||||
if (processed) return;
|
||||
processed = true;
|
||||
|
||||
const buildNumber = process.env.LLAMA_BUILD_NUMBER;
|
||||
if (!buildNumber) return;
|
||||
const buildNumber = process.env.LLAMA_BUILD_NUMBER || 'b0000';
|
||||
|
||||
const outDir = resolve(OUTPUT_DIR);
|
||||
const indexPath = resolve(outDir, 'index.html');
|
||||
|
||||
Reference in New Issue
Block a user