lscpu
Architecture:                aarch64
  CPU op-mode(s):            32-bit, 64-bit
  Byte Order:                Little Endian
CPU(s):                      4
  On-line CPU(s) list:       0-3
Vendor ID:                   ARM
  Model name:                Neoverse-N1
    Model:                   1
    Thread(s) per core:      1
    Core(s) per cluster:     4
    Socket(s):               -
    Cluster(s):              1
    Stepping:                r3p1
    BogoMIPS:                50.00
    Flags:                   fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics 
                             fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp
NUMA:                        
  NUMA node(s):              1
  NUMA node0 CPU(s):         0-3
Vulnerabilities:             
  Gather data sampling:      Not affected
  Indirect target selection: Not affected
  Itlb multihit:             Not affected
  L1tf:                      Not affected
  Mds:                       Not affected
  Meltdown:                  Not affected
  Mmio stale data:           Not affected
  Reg file data sampling:    Not affected
  Retbleed:                  Not affected
  Spec rstack overflow:      Not affected
  Spec store bypass:         Mitigation; Speculative Store Bypass disabled via p
                             rctl
  Spectre v1:                Mitigation; __user pointer sanitization
  Spectre v2:                Mitigation; CSV2, BHB
  Srbds:                     Not affected
  Tsa:                       Not affected
  Tsx async abort:           Not affected
  Vmscape:                   Not affected

sudo apt update && sudo apt upgrade -y
sudo apt install -y build-essential \
  cmake \
  git \
  wget \
  curl \
  python3 \
  python3-venv \
  python3-pip \
  libopenblas-dev \
  pkg-config \
  unzip

# 切换至 root 用户执行
sudo -i
# 临时生效
ulimit -l unlimited
# 写入配置文件以永久生效
echo "* soft memlock unlimited" | sudo tee -a /etc/security/limits.conf
echo "* hard memlock unlimited" | sudo tee -a /etc/security/limits.conf

# 拉取最新源码
git clone https://github.com/ggml-org/llama.cpp.git
cd llama.cpp

# 编译配置
cmake .. \
  -DCMAKE_BUILD_TYPE=Release \
  -DGGML_NATIVE=ON \
  -DGGML_OPENMP=ON \
  -DGGML_BLAS=ON \
  -DGGML_BLAS_VENDOR=OpenBLAS
# 开始编译
cmake --build . --config Release -j4

sudo cp bin/llama-server /usr/local/bin/
sudo cp bin/llama-quantize /usr/local/bin/ 2>/dev/null || true
sudo chmod +x /usr/local/bin/llama-server
# 确保下面这行在 Shell 配置中
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
source ~/.bashrc

python3 -m pip install --user --force-reinstall "huggingface_hub[cli]" hf-xet --break-system-packages
# 模型存储目录
mkdir -p ~/models/gemma-4-26b && cd ~/models/gemma-4-26b
# 下载模型
hf download unsloth/gemma-4-26B-A4B-it-GGUF \
  --local-dir . \
  --include "*UD-IQ4_NL.gguf" \
  --repo-type model
# 查看模型
ls -lh *.gguf

llama-server \
  -m gemma-4-26B-A4B-it-UD-Q4_K_M.gguf \
  --host 100.64.0.6 \
  --port 8080 \
  --api-key your-super-secret-key-1234567890abcdef \
  -t 4 \
  -tb 4 \
  -c 8192 \
  --mlock \
  --no-mmap \
  --parallel 1 \
  --batch-size 512 \
  --ubatch-size 512 \
  --cache-type-k q8_0 \
  --cache-type-v q8_0 \
  --no-warmup

curl http://100.64.0.6:8080/v1/chat/completions \
  -H "Authorization: Bearer your-super-secret-key-1234567890abcdef" \
  -H "Content-Type: application/json" \
  -d '{"model":"gemma-4-26B-A4B-it","messages":[{"role":"user","content":"Hello, pls introduce yourself, who are you? what can you do?When using the `4-26B-A4B-it-UD-Q4_K_M.gguf` model on an Oracle ARM 4C 24GB VPS, what input and output token processing speeds would you expect?"}],"temperature":0.7}'

{"choices":[{"finish_reason":"stop","index":0,"message":{"role":"assistant","content":"### Part 1: Introduction\n\n**Who am I?**\nI am a large language model, trained by Google. You can think of me as a sophisticated AI assistant capable of processing, reasoning, and generating human-like text across a vast array of subjects.\n\n**What can I do?**\nMy capabilities are diverse and include, but are not limited to:\n*   **Content Generation:** Writing essays, emails, stories, poems, and scripts.\n*   **Coding & Technical Tasks:** Writing, debugging, and explaining code in dozens of programming languages.\n*   **Reasoning & Problem Solving:** Solving mathematical problems, logical puzzles, and providing step-by-step explanations.\n*   **Summarization & Analysis:** Condensing long documents into key points or analyzing the sentiment and tone of a text.\n*   **Translation:** Translating text between dozens of major world languages.\n*   **Information Retrieval:** Answering questions on history, science, culture, and more (within my training data limits).\n\n---\n\n### Part 2: Performance Estimation\n\nYou are asking about a very specific setup: a **Mixture-of-Experts (MoE)** model (`4-26B-A4B`) running on **Oracle ARM (Ampere Altra)** hardware via **GGUF** (llama.cpp).\n\n#### 1. Understanding the Model & Hardware\n*   **The Model (`4-26B-A4B`):** This is an MoE model. While it has **26 billion total parameters**, it only uses **4 billion active parameters** per token generated. This is crucial: your speed will be dictated by the \"active\" parameter count for compute, but by the \"total\" parameter count for memory loading.\n*   **Quantization (`Q4_K_M`):** A 4-bit quantization is efficient. A 26B model at 4-bit will take up approximately **14–16 GB of RAM**.\n*   **The Hardware (Oracle ARM 4C 24GB):**\n    *   **RAM:** 24GB is plenty to hold the 15GB model plus a healthy KV cache.\n    *   **CPU:** 4 ARM cores (Ampere Altra). These are high-performance server cores, but 4 cores is a relatively low count for a model of this size.\n    *   **Memory Bandwidth:** This is your primary bottleneck. In a VPS environment, memory bandwidth is often shared or limited compared to a dedicated bare-metal server.\n\n#### 2. Estimated Performance Metrics\n\nSince I cannot run a live benchmark on your specific VPS instance, I will provide an estimate based on the architectural characteristics of MoE models on ARM-based cloud CPUs.\n\n**A. Output Token Processing Speed (Decoding/Generation)**\n*   **Nature:** This is **Memory Bandwidth Bound**. The CPU must pull the active weights from the RAM for every single token generated.\n*   **The MoE Advantage:** Because only 4B parameters are active, the CPU doesn't have to do the math for all 26B, but it still has to navigate the routing logic.\n*   **Estimated Speed:** **3 – 8 tokens per second (t/s).**\n    *   *Why?* Even though only 4B parameters are \"active,\" the overhead of the MoE routing and the relatively modest memory bandwidth of a 4-core ARM VPS will prevent high-speed generation. You will likely experience a \"reading speed\" (similar to a human reading), which is usable but not \"instant.\"\n\n**B. Input Token Processing Speed (Prefill/Prompt Processing)**\n*   **Nature:** This is **Compute Bound**. This is the stage where the model processes your initial prompt to build the KV cache.\n*   **The ARM Advantage:** Ampere Altra cores have excellent instruction sets for math, but 4 cores is a tight limit for large matrix-matrix multiplications.\n*   **Estimated Speed:** **150 – 400 tokens per second.**\n    *   *Why?* Prefill is much faster than generation because it processes tokens in parallel batches. However, because you only have 4 cores, the parallelization is limited compared to a 32-core or 64-core instance.\n\n#### Summary Table (Estimated)\n\n| Metric | Estimated Value | Bottleneck |\n| :--- | :--- | :--- |\n| **Prompt Processing (Prefill)** | 150 – 400 t/s | CPU Compute (Cores) |\n| **Token Generation (Decoding)** | 3 – 8 t/s | Memory Bandwidth |\n| **RAM Usage** | ~15GB - 17GB | Model Size + KV Cache |\n\n#### Tips for Optimization on your Oracle VPS:\n1.  **Use `llama.cpp` with OpenBLAS or ARM NEON:** Ensure your build of `llama.cpp` is optimized for ARM NEON instructions to maximize the efficiency of those 4 cores.\n2.  **Thread Tuning:** Since you have 4 cores, set your threads (`-t`) to **4**. Do not over-subscribe threads, as this will increase context-switching latency and slow down generation.\n3.  **Flash Attention:** If using a recent version of `llama.cpp`, enable Flash Attention to reduce the memory footprint and speed up the prefill stage.","reasoning_content":"\n    *   Who am I? (Identity/Introduction).\n    *   What can I do? (Capabilities).\n    *   Technical Question: Performance estimation for a specific model (`4-26B-A4B-it-UD-Q4_K_M.gguf`) on specific hardware (Oracle ARM 4C 24GB VPS).\n    *   Metrics requested: Input token processing speed (prefill) and output token processing speed (decoding/generation).\n\n        *   *Identity:* I am a large language model trained by Google.\n        *   *Capabilities:* Text generation, reasoning, coding, translation, summarization, answering questions, etc.\n\n        *   *Model:* `4-26B-A4B-it-UD-Q4_K_M.gguf`.\n            *   \"4-26B\": This looks like a Mixture-of-Experts (MoE) model. Total parameters = 26B, Active parameters (A) = 4B.\n            *   \"UD\": Likely \"Uncensored\" or a specific fine-tune variant.\n            *   \"Q4_K_M\": 4-bit quantization (K-quantization, medium).\n            *   \"GGUF\": Format for llama.cpp/llama-cpp-python.\n        *   *Hardware:* Oracle ARM 4C 24GB VPS.\n            *   CPU: Ampere Altra (typically) - 4 ARM cores.\n            *   RAM: 24GB.\n            *   Architecture: ARM64.\n        *   *Bottlenecks:*\n            *   *Memory Bandwidth:* This is the primary bottleneck for LLM inference. ARM VPSs (like Oracle's) usually have decent memory bandwidth compared to standard cloud CPUs, but it's significantly lower than GPU VRAM (H100/A100) or even Apple Silicon Unified Memory.\n            *   *Compute:* MoE models are much faster at generation because they only use a fraction of parameters per token (4B vs 26B). However, the *entire* 26B model must reside in RAM.\n            *   *RAM Check:* A 26B model at Q4_K_M quantization will take roughly: $26 \\times 10^9 \\times 0.5$ (bits to bytes approx) $\\approx 13-15$ GB. With 24GB RAM, this fits comfortably with room for KV cache.\n\n        *   *Step 1: Estimate Model Size.*\n            26B parameters at 4-bit $\\approx 14-15$ GB.\n            Fits in 24GB RAM.\n\n        *   *Step 2: Estimate Generation Speed (Decoding).*\n            Decoding is memory-bandwidth bound.\n            Formula: $\\text{Tokens per second (t/s)} \\approx \\frac{\\text{Memory Bandwidth (GB/s)}}{\\text{Model Size (GB)} \\times \\text{Active Parameters Factor}}$.\n            *Actually, for MoE, the weight loading is for the active parameters, but you still have to access the routing and the loaded weights.*\n            Wait, in MoE, you only load the active weights from RAM to cache for each token.\n            Active parameters = 4B.\n            Memory bandwidth for Oracle ARM (Ampere Altra) is roughly 50-100 GB/s (estimated based on typical quad-channel/dual-channel setups in cloud environments).\n            If we use 4B active parameters at 4-bit: $4 \\times 0.5 = 2$ GB per token.\n            If bandwidth is 50 GB/s: $50 / 2 = 25$ t/s.\n            *However*, MoE overhead and the fact that we are on a CPU (not a specialized accelerator) reduces this. Real-world CPU inference for a 4B active MoE is usually around 5-15 t/s on decent hardware.\n\n        *   *Step 3: Estimate Prefill Speed (Input).*\n            Prefill is compute-bound (matrix-matrix multiplication).\n            ARM cores are decent but not as fast as GPUs for large matrices.\n            Expectation: Roughly 100-500 tokens per second depending on prompt length and core utilization.\n\n        *   *Disclaimer:* I am an AI, not a benchmark tool. These are estimates.\n        *   *The Model:* Analyze the MoE nature.\n        *   *The Hardware:* Analyze ARM/RAM.\n        *   *The Estimation (Decoding):* Focus on the 4B active parameters.\n        *   *The Estimation (Prefill):* Focus on the compute capability.\n        *   *Summary Table/List.*\n\n    *   *Correction:* I shouldn't say it's definitely 25 t/s. That's too optimistic for a VPS. VPSs often have \"noisy neighbors\" and throttled memory bandwidth.\n    *   *Reality Check:* A 4B active model on 4 ARM cores... 4 cores is quite low for a 26B model (even if MoE). The bottleneck will be the single-threaded performance and the memory latency.\n    *   *Revised Estimate:*\n        *   Decoding: 3-8 tokens/sec.\n        *   Prefill: 100-300 tokens/sec."}}],"created":1775532629,"model":"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf","system_fingerprint":"b8683-d0a6dfeb2","object":"chat.completion","usage":{"completion_tokens":2347,"prompt_tokens":83,"total_tokens":2430,"prompt_tokens_details":{"cached_tokens":0}},"id":"chatcmpl-zwHxF7ycjfCWAiyMA3cjw2zXRKsckpqB","timings":{"cache_n":0,"prompt_n":83,"prompt_ms":3486.44,"prompt_per_token_ms":42.005301204819276,"prompt_per_second":23.80651897064054,"predicted_n":2347,"predicted_ms":311498.306,"predicted_per_token_ms":132.72190285470813,"predicted_per_second":7.534551407801236}}

prompt eval time =    3486.44 ms /    83 tokens (   42.01 ms per token,    23.81 tokens per second)
       eval time =  311498.31 ms /  2347 tokens (  132.72 ms per token,     7.53 tokens per second)
      total time =  314984.75 ms /  2430 tokens

sudo vim /etc/systemd/system/llama.service

[Unit]
Description=Gemma 4 26B API Server (Tailscale Network)
After=network-online.target tailscaled.service
Wants=network-online.target tailscaled.service

[Service]
Type=simple
User=dejavu
# 修改为绝对路径
WorkingDirectory=/home/dejavu/models/gemma-4-26b
Environment="PATH=/usr/local/bin:/usr/bin:/bin"

ExecStart=/usr/local/bin/llama-server \
  -m /home/dejavu/models/gemma-4-26b/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf \
  --host 100.64.0.6 \
  --port 8080 \
  --api-key your-super-secret-key-1234567890abcdef \
  -t 4 \
  -tb 4 \
  -c 8192 \
  --mlock \
  --no-mmap \
  --parallel 1 \
  --batch-size 512 \
  --ubatch-size 512 \
  --cache-type-k q8_0 \
  --cache-type-v q8_0 \
  --no-warmup

Restart=always
RestartSec=10

StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=gemma-api

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable llama --now
sudo systemctl status llama
# 查看实时日志
sudo journalctl -u llama -f