mirror of
https://github.com/ruvnet/RuView
synced 2026-07-25 17:51:48 +00:00
feat: HuggingFace model publishing pipeline + model card
- publish-huggingface.sh: retrieves HF token from GCloud Secrets, uploads models to ruvnet/wifi-densepose-pretrained - publish-huggingface.py: Python alternative with --dry-run support - docs/huggingface/MODEL_CARD.md: beginner-friendly model card with WiFi sensing explanation, quick start code, hardware BOM, and citation GCloud Secret: HUGGINGFACE_API_KEY in project cognitum-20260110 Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
ruv
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Publish WiFi-DensePose pre-trained models to HuggingFace Hub.
|
||||
|
||||
Retrieves the HuggingFace API token from Google Cloud Secrets,
|
||||
then uploads model files from dist/models/ to a HuggingFace repo.
|
||||
|
||||
Prerequisites:
|
||||
- gcloud CLI authenticated with access to cognitum-20260110
|
||||
- pip install huggingface_hub google-cloud-secret-manager
|
||||
|
||||
Usage:
|
||||
python scripts/publish-huggingface.py
|
||||
python scripts/publish-huggingface.py --repo ruvnet/wifi-densepose-pretrained --version v0.5.4
|
||||
python scripts/publish-huggingface.py --dry-run
|
||||
python scripts/publish-huggingface.py --token hf_xxxxx # skip GCloud lookup
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
EXPECTED_FILES = [
|
||||
"pretrained-encoder.onnx",
|
||||
"pretrained-heads.onnx",
|
||||
"pretrained.rvf",
|
||||
"room-profiles.json",
|
||||
"collection-witness.json",
|
||||
"config.json",
|
||||
"README.md",
|
||||
]
|
||||
|
||||
|
||||
def get_token_from_gcloud(
|
||||
project: str = "cognitum-20260110",
|
||||
secret: str = "HUGGINGFACE_API_KEY",
|
||||
) -> str:
|
||||
"""Retrieve HuggingFace token from Google Cloud Secret Manager."""
|
||||
# Try the gcloud CLI first (simpler, no extra deps)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"gcloud", "secrets", "versions", "access", "latest",
|
||||
f"--secret={secret}",
|
||||
f"--project={project}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return result.stdout.strip()
|
||||
except FileNotFoundError:
|
||||
pass # gcloud not installed, try Python SDK
|
||||
|
||||
# Fall back to the Python SDK
|
||||
try:
|
||||
from google.cloud import secretmanager
|
||||
|
||||
client = secretmanager.SecretManagerServiceClient()
|
||||
name = f"projects/{project}/secrets/{secret}/versions/latest"
|
||||
response = client.access_secret_version(request={"name": name})
|
||||
return response.payload.data.decode("utf-8").strip()
|
||||
except ImportError:
|
||||
print(
|
||||
"ERROR: Neither gcloud CLI nor google-cloud-secret-manager is available.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("Install: pip install google-cloud-secret-manager", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as exc:
|
||||
print(f"ERROR: Failed to retrieve secret: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def auto_version() -> str:
|
||||
"""Detect version from git describe."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "describe", "--tags", "--always"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return "dev"
|
||||
|
||||
|
||||
def validate_model_dir(model_dir: Path) -> list[Path]:
|
||||
"""List available files and warn about missing expected files."""
|
||||
found: list[Path] = []
|
||||
missing: list[str] = []
|
||||
|
||||
for fname in EXPECTED_FILES:
|
||||
path = model_dir / fname
|
||||
if path.is_file():
|
||||
size = path.stat().st_size
|
||||
print(f" [OK] {fname} ({size:,} bytes)")
|
||||
found.append(path)
|
||||
else:
|
||||
print(f" [MISSING] {fname}")
|
||||
missing.append(fname)
|
||||
|
||||
# Also pick up any extra files not in the expected list
|
||||
for path in sorted(model_dir.iterdir()):
|
||||
if path.is_file() and path.name not in EXPECTED_FILES:
|
||||
size = path.stat().st_size
|
||||
print(f" [EXTRA] {path.name} ({size:,} bytes)")
|
||||
found.append(path)
|
||||
|
||||
if missing:
|
||||
print(f"\nWARNING: {len(missing)} expected file(s) missing.")
|
||||
print("Upload will proceed with available files.\n")
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def publish(
|
||||
repo_id: str,
|
||||
model_dir: Path,
|
||||
version: str,
|
||||
token: str,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""Upload model files to HuggingFace Hub."""
|
||||
try:
|
||||
from huggingface_hub import HfApi, login
|
||||
except ImportError:
|
||||
print("Installing huggingface_hub...")
|
||||
subprocess.check_call(
|
||||
[sys.executable, "-m", "pip", "install", "--quiet", "huggingface_hub"]
|
||||
)
|
||||
from huggingface_hub import HfApi, login
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Repo: https://huggingface.co/{repo_id}")
|
||||
print(f"Version: {version}")
|
||||
print(f"Model dir: {model_dir}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
print("Validating model files...")
|
||||
files = validate_model_dir(model_dir)
|
||||
|
||||
if not files:
|
||||
print("ERROR: No files to upload.")
|
||||
sys.exit(1)
|
||||
|
||||
if dry_run:
|
||||
print(f"\n[DRY RUN] Would upload {len(files)} file(s) to {repo_id}")
|
||||
for f in files:
|
||||
print(f" - {f.name}")
|
||||
print(f"[DRY RUN] Version tag: {version}")
|
||||
return
|
||||
|
||||
print("Authenticating with HuggingFace...")
|
||||
login(token=token, add_to_git_credential=False)
|
||||
api = HfApi()
|
||||
|
||||
print("Creating repo (if needed)...")
|
||||
api.create_repo(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
exist_ok=True,
|
||||
private=False,
|
||||
)
|
||||
|
||||
print("Uploading files...")
|
||||
commit_info = api.upload_folder(
|
||||
folder_path=str(model_dir),
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
commit_message=f"Upload WiFi-DensePose pretrained models ({version})",
|
||||
)
|
||||
|
||||
# Tag
|
||||
try:
|
||||
api.create_tag(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
tag=version,
|
||||
tag_message=f"WiFi-DensePose pretrained models {version}",
|
||||
)
|
||||
print(f"Tagged as: {version}")
|
||||
except Exception as exc:
|
||||
print(f"Tag '{version}' may already exist: {exc}")
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Published successfully!")
|
||||
print(f"URL: https://huggingface.co/{repo_id}")
|
||||
print(f"Version: {version}")
|
||||
print(f"Commit: {commit_info.commit_url}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Publish WiFi-DensePose models to HuggingFace Hub",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo",
|
||||
default="ruvnet/wifi-densepose-pretrained",
|
||||
help="HuggingFace repo ID (default: ruvnet/wifi-densepose-pretrained)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
default="",
|
||||
help="Version tag (default: auto from git describe)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-dir",
|
||||
default="dist/models",
|
||||
help="Directory containing model files (default: dist/models)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
default="cognitum-20260110",
|
||||
help="GCloud project ID (default: cognitum-20260110)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--secret",
|
||||
default="HUGGINGFACE_API_KEY",
|
||||
help="GCloud secret name (default: HUGGINGFACE_API_KEY)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
default="",
|
||||
help="HuggingFace token (skip GCloud lookup if provided)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Preview upload without actually uploading",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
model_dir = Path(args.model_dir)
|
||||
version = args.version or auto_version()
|
||||
|
||||
if not model_dir.is_dir():
|
||||
print(f"ERROR: Model directory does not exist: {model_dir}")
|
||||
print("Create it and populate with model files first.")
|
||||
sys.exit(1)
|
||||
|
||||
# Get token
|
||||
if args.dry_run:
|
||||
token = "dry-run-no-token-needed"
|
||||
elif args.token:
|
||||
token = args.token
|
||||
else:
|
||||
print(f"Retrieving HuggingFace token from GCloud ({args.project})...")
|
||||
token = get_token_from_gcloud(project=args.project, secret=args.secret)
|
||||
print("Token retrieved.")
|
||||
|
||||
publish(
|
||||
repo_id=args.repo,
|
||||
model_dir=model_dir,
|
||||
version=version,
|
||||
token=token,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/bin/bash
|
||||
# Publish WiFi-DensePose pre-trained models to HuggingFace Hub
|
||||
#
|
||||
# Retrieves the HuggingFace API token from Google Cloud Secrets,
|
||||
# then uploads model files from dist/models/ to a HuggingFace repo.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - gcloud CLI authenticated with access to cognitum-20260110
|
||||
# - Python 3.8+ with pip
|
||||
# - Model files present in dist/models/
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/publish-huggingface.sh
|
||||
# bash scripts/publish-huggingface.sh --repo ruvnet/wifi-densepose-pretrained --version v0.5.4
|
||||
# bash scripts/publish-huggingface.sh --dry-run
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------- defaults ----------
|
||||
REPO="ruvnet/wifi-densepose-pretrained"
|
||||
VERSION=""
|
||||
GCLOUD_PROJECT="cognitum-20260110"
|
||||
SECRET_NAME="HUGGINGFACE_API_KEY"
|
||||
MODEL_DIR="dist/models"
|
||||
DRY_RUN=false
|
||||
|
||||
# ---------- parse args ----------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--repo) REPO="$2"; shift 2 ;;
|
||||
--version) VERSION="$2"; shift 2 ;;
|
||||
--model-dir) MODEL_DIR="$2"; shift 2 ;;
|
||||
--project) GCLOUD_PROJECT="$2"; shift 2 ;;
|
||||
--secret) SECRET_NAME="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help)
|
||||
echo "Usage: bash scripts/publish-huggingface.sh [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --repo REPO HuggingFace repo (default: ruvnet/wifi-densepose-pretrained)"
|
||||
echo " --version VERSION Version tag (default: auto from git describe)"
|
||||
echo " --model-dir DIR Model directory (default: dist/models)"
|
||||
echo " --project PROJECT GCloud project (default: cognitum-20260110)"
|
||||
echo " --secret SECRET GCloud secret name (default: HUGGINGFACE_API_KEY)"
|
||||
echo " --dry-run Show what would be uploaded without uploading"
|
||||
echo " -h, --help Show this help"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Unknown option: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------- auto-detect version ----------
|
||||
if [ -z "$VERSION" ]; then
|
||||
VERSION=$(git describe --tags --always 2>/dev/null || echo "dev")
|
||||
echo "Auto-detected version: ${VERSION}"
|
||||
fi
|
||||
|
||||
# ---------- validate model files ----------
|
||||
EXPECTED_FILES=(
|
||||
"pretrained-encoder.onnx"
|
||||
"pretrained-heads.onnx"
|
||||
"pretrained.rvf"
|
||||
"room-profiles.json"
|
||||
"collection-witness.json"
|
||||
"config.json"
|
||||
"README.md"
|
||||
)
|
||||
|
||||
echo "=== WiFi-DensePose HuggingFace Publisher ==="
|
||||
echo "Repo: ${REPO}"
|
||||
echo "Version: ${VERSION}"
|
||||
echo "Model dir: ${MODEL_DIR}"
|
||||
echo ""
|
||||
|
||||
MISSING=0
|
||||
for f in "${EXPECTED_FILES[@]}"; do
|
||||
if [ -f "${MODEL_DIR}/${f}" ]; then
|
||||
SIZE=$(stat --printf="%s" "${MODEL_DIR}/${f}" 2>/dev/null || stat -f "%z" "${MODEL_DIR}/${f}" 2>/dev/null || echo "?")
|
||||
echo " [OK] ${f} (${SIZE} bytes)"
|
||||
else
|
||||
echo " [MISSING] ${f}"
|
||||
MISSING=$((MISSING + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$MISSING" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "WARNING: ${MISSING} expected file(s) missing from ${MODEL_DIR}/"
|
||||
echo "The upload will proceed with available files only."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Count actual files to upload
|
||||
FILE_COUNT=$(find "${MODEL_DIR}" -maxdepth 1 -type f | wc -l)
|
||||
if [ "$FILE_COUNT" -eq 0 ]; then
|
||||
echo "ERROR: No files found in ${MODEL_DIR}/. Nothing to upload."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---------- dry run ----------
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo ""
|
||||
echo "[DRY RUN] Would upload ${FILE_COUNT} files to https://huggingface.co/${REPO}"
|
||||
echo "[DRY RUN] Files:"
|
||||
find "${MODEL_DIR}" -maxdepth 1 -type f -exec basename {} \; | sort | while read -r fname; do
|
||||
echo " - ${fname}"
|
||||
done
|
||||
echo "[DRY RUN] Version tag: ${VERSION}"
|
||||
echo ""
|
||||
echo "Run without --dry-run to actually upload."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------- retrieve HuggingFace token ----------
|
||||
echo ""
|
||||
echo "Retrieving HuggingFace token from GCloud Secrets..."
|
||||
HF_TOKEN=$(gcloud secrets versions access latest \
|
||||
--secret="${SECRET_NAME}" \
|
||||
--project="${GCLOUD_PROJECT}" 2>/dev/null)
|
||||
|
||||
if [ -z "$HF_TOKEN" ]; then
|
||||
echo "ERROR: Failed to retrieve secret '${SECRET_NAME}' from project '${GCLOUD_PROJECT}'."
|
||||
echo "Make sure you are authenticated: gcloud auth login"
|
||||
echo "And have access to the secret: gcloud secrets list --project=${GCLOUD_PROJECT}"
|
||||
exit 1
|
||||
fi
|
||||
echo "Token retrieved successfully."
|
||||
|
||||
# ---------- install huggingface_hub if needed ----------
|
||||
if ! python3 -c "import huggingface_hub" 2>/dev/null; then
|
||||
echo "Installing huggingface_hub..."
|
||||
pip3 install --quiet huggingface_hub
|
||||
fi
|
||||
|
||||
# ---------- upload via Python ----------
|
||||
echo ""
|
||||
echo "Uploading to https://huggingface.co/${REPO} ..."
|
||||
|
||||
python3 - <<PYEOF
|
||||
import os
|
||||
from huggingface_hub import HfApi, login
|
||||
|
||||
token = os.environ.get("HF_TOKEN_OVERRIDE") or """${HF_TOKEN}"""
|
||||
repo_id = "${REPO}"
|
||||
model_dir = "${MODEL_DIR}"
|
||||
version = "${VERSION}"
|
||||
|
||||
login(token=token, add_to_git_credential=False)
|
||||
api = HfApi()
|
||||
|
||||
# Create repo if it doesn't exist
|
||||
api.create_repo(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
exist_ok=True,
|
||||
private=False,
|
||||
)
|
||||
|
||||
# Upload the entire folder
|
||||
commit_info = api.upload_folder(
|
||||
folder_path=model_dir,
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
commit_message=f"Upload WiFi-DensePose pretrained models ({version})",
|
||||
)
|
||||
|
||||
# Create a tag for this version
|
||||
try:
|
||||
api.create_tag(
|
||||
repo_id=repo_id,
|
||||
repo_type="model",
|
||||
tag=version,
|
||||
tag_message=f"WiFi-DensePose pretrained models {version}",
|
||||
)
|
||||
print(f"Tagged as: {version}")
|
||||
except Exception as e:
|
||||
print(f"Tag '{version}' may already exist: {e}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print(f"Published successfully!")
|
||||
print(f"URL: https://huggingface.co/{repo_id}")
|
||||
print(f"Version: {version}")
|
||||
print(f"Commit: {commit_info.commit_url}")
|
||||
print("=" * 60)
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "Done."
|
||||
Reference in New Issue
Block a user