LiquidAIのCPUパフォーマンス検証 LFM2-350M vs Qwen2.5-0.5B vs Qwen3-0.6B

!pip install -U torch transformers accelerate datasets evaluate sacrebleu rouge-score psutil matplotlib pynvml

import sys, platform, torch, datetime, os
print("Python:", sys.version.split()[0])
print("PyTorch:", torch.__version__)
print("CUDA available:", torch.cuda.is_available())
print("Device name:", torch.cuda.get_device_name(0) if torch.cuda.is_available() else "CPU")
print("Started at:", datetime.datetime.now())

import matplotlib.pyplot as plt

Python: 3.10.5
PyTorch: 2.9.0
CUDA available: False
Device name: CPU
Started at: 2025-10-31 16:47:35.165621

LFM_MODEL_NAME  = "LiquidAI/LFM2-350M"
QWEN_MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"

DTYPE = "float32"          
COMPUTE_DEVICE = "cpu"      
MAX_NEW_TOKENS = 256
WARMUP_NEW_TOKENS = 8
PROMPT_LENGTHS = [512, 2048, 8192]

SEED = 42

PROMPT_CHUNK = (
    "生成AIの推論最適化では、KVキャッシュ、量子化、動的バッチングなどの手法が活用され、"
    "入力長に応じたメモリ効率とレイテンシのトレードオフが重要とされます。"
    "高負荷時における安定性とスループットの確保は、現場導入を進める上での鍵となります。"
)

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed

def resolve_dtype(dtype_str):
    if dtype_str == "auto":
        return None
    if not hasattr(torch, dtype_str):
        raise ValueError(f"Unsupported dtype: {dtype_str}")
    return getattr(torch, dtype_str)

def load_model_and_tokenizer(name: str, dtype="float32", device="cpu"):
    tokenizer = AutoTokenizer.from_pretrained(name)
    dtype_obj = resolve_dtype(dtype)
    model_kwargs = {}
    if dtype_obj is not None:
        model_kwargs["torch_dtype"] = dtype_obj
    model = AutoModelForCausalLM.from_pretrained(name, **model_kwargs)
    if dtype_obj is not None:
        model = model.to(dtype=dtype_obj)
    model = model.to(device)
    return tokenizer, model

set_seed(SEED)
lfm_tok, lfm_model   = load_model_and_tokenizer(LFM_MODEL_NAME, DTYPE, COMPUTE_DEVICE)
qwen_tok, qwen_model = load_model_and_tokenizer(QWEN_MODEL_NAME, DTYPE, COMPUTE_DEVICE)
print(f"Models loaded on {COMPUTE_DEVICE} with dtype={DTYPE}")

import threading, time, psutil, queue, os, sys, subprocess, re, math

class SysMonitor:
    def __init__(self, interval=0.1):
        self.interval = interval
        self.q = queue.Queue()
        self._stop = threading.Event()
        self._th = None
        self.proc = psutil.Process(os.getpid())
        self._platform = sys.platform
        try:
            self._page_size = os.sysconf('SC_PAGE_SIZE')
        except (AttributeError, ValueError):
            self._page_size = 4096

    def _linux_smaps(self):
        data = {}
        path = f"/proc/{self.proc.pid}/smaps_rollup"
        try:
            with open(path) as fh:
                for line in fh:
                    if line.startswith('Pss:'):
                        data['smaps_pss_bytes'] = int(line.split()[1]) * 1024
                    elif line.startswith('FilePss:'):
                        data['smaps_file_pss_bytes'] = int(line.split()[1]) * 1024
                    elif line.startswith('Swap:'):
                        data['smaps_swap_bytes'] = int(line.split()[1]) * 1024
        except (FileNotFoundError, PermissionError):
            pass
        return data

    def _darwin_vm_stat(self):
        data = {}
        try:
            out = subprocess.check_output(['vm_stat'], text=True)
        except Exception:
            return data
        page_size = self._page_size
        for line in out.splitlines():
            m = re.search(r'page size of (\d+)', line)
            if m:
                page_size = int(m.group(1))
                continue
            m = re.match(r'"?([A-Za-z\- ]+)"?:\s+([\d\.]+)', line)
            if not m:
                continue
            key = m.group(1).strip().strip('"')
            val = int(float(m.group(2)))
            if key == 'File-backed pages':
                data['file_backed_bytes'] = val * page_size
            elif key == 'Anonymous pages':
                data['anonymous_bytes'] = val * page_size
        return data

    def _collect_os_memory(self):
        if self._platform.startswith('linux'):
            return self._linux_smaps()
        if self._platform == 'darwin':
            return self._darwin_vm_stat()
        return {}

    def _run(self):
        while not self._stop.is_set():
            ts = time.time()
            try:
                full = self.proc.memory_full_info()
            except psutil.Error:
                full = self.proc.memory_info()
            vm = psutil.virtual_memory()
            swap = psutil.swap_memory()
            cpu = psutil.cpu_percent(interval=None)
            sample = {
                'ts': ts,
                'vm_percent': vm.percent,
                'rss': getattr(full, 'rss', None),
                'uss': getattr(full, 'uss', None),
                'cpu_percent': cpu,
                'swap_used': getattr(swap, 'used', 0),
            }
            sample.update(self._collect_os_memory())
            self.q.put(sample)
            time.sleep(self.interval)

    def start(self):
        psutil.cpu_percent(interval=None)  
        self._stop.clear()
        self._th = threading.Thread(target=self._run, daemon=True)
        self._th.start()

    def stop(self):
        self._stop.set()
        if self._th:
            self._th.join()

    def collect(self):
        out = []
        while not self.q.empty():
            out.append(self.q.get())
        return out


def plot_mem(series, title="System Memory Usage"):
    if not series:
        print("No samples.")
        return
    import matplotlib.pyplot as plt

    t0 = series[0]['ts']
    t = [s['ts'] - t0 for s in series]

    fig, axes = plt.subplots(2, 2, figsize=(12, 8))
    axes = axes.flatten()

    axes[0].plot(t, [s['vm_percent'] for s in series])
    axes[0].set_xlabel('Time [s]')
    axes[0].set_ylabel('System Memory [%]')
    axes[0].set_title(f"{title} - System %")

    rss = [s.get('rss') / (1024 ** 3) if s.get('rss') is not None else float('nan') for s in series]
    uss = [s.get('uss') / (1024 ** 3) if s.get('uss') is not None else float('nan') for s in series]
    axes[1].plot(t, rss, label='RSS')
    if any(not math.isnan(v) for v in uss):
        axes[1].plot(t, uss, label='USS')
    axes[1].set_xlabel('Time [s]')
    axes[1].set_ylabel('Process Memory [GB]')
    axes[1].set_title(f"{title} - Process RSS/USS")
    if axes[1].lines:
        axes[1].legend()

    swap = [s.get('swap_used', 0) / (1024 ** 3) for s in series]
    axes[2].plot(t, swap)
    axes[2].set_xlabel('Time [s]')
    axes[2].set_ylabel('Swap Used [GB]')
    axes[2].set_title(f"{title} - Swap Usage")

    file_key = 'smaps_file_pss_bytes' if any('smaps_file_pss_bytes' in s for s in series) else 'file_backed_bytes'
    file_vals = [s.get(file_key, 0) / (1024 ** 3) for s in series]
    anon_vals = [s.get('anonymous_bytes', 0) / (1024 ** 3) for s in series]
    pss_vals = [s.get('smaps_pss_bytes', 0) / (1024 ** 3) for s in series]
    if any(v > 0 for v in file_vals + anon_vals + pss_vals):
        if any(v > 0 for v in file_vals):
            axes[3].plot(t, file_vals, label='File-backed')
        if any(v > 0 for v in anon_vals):
            axes[3].plot(t, anon_vals, label='Anonymous')
        if any(v > 0 for v in pss_vals):
            axes[3].plot(t, pss_vals, label='PSS')
        axes[3].set_xlabel('Time [s]')
        axes[3].set_ylabel('Memory [GB]')
        axes[3].set_title(f"{title} - Mapped Pages")
        if axes[3].lines:
            axes[3].legend()
    else:
        axes[3].set_axis_off()

    plt.tight_layout()
    plt.show()

import time
import torch
import pandas as pd
from IPython.display import display


def build_prompt_text(target_tokens, base_chunk=PROMPT_CHUNK, approx_chars_per_token=4):
    needed_chars = target_tokens * approx_chars_per_token
    repeat = (needed_chars // (len(base_chunk) + 1)) + 2
    prompt = ((base_chunk + "\n") * repeat)[:needed_chars]
    return prompt


def summarize_series(series, key):
    vals = [s.get(key) for s in series if s.get(key) is not None]
    return max(vals) if vals else None


def generate_benchmark(model, tokenizer, prompt_text, max_new_tokens=128, warmup_new_tokens=8):
    device = next(model.parameters()).device
    inputs = tokenizer(prompt_text, return_tensors="pt")
    input_tokens = inputs["input_ids"].shape[-1]
    inputs = {k: v.to(device) for k, v in inputs.items()}

    if warmup_new_tokens:
        with torch.no_grad():
            model.generate(**inputs, max_new_tokens=warmup_new_tokens, do_sample=False, use_cache=True)
            if torch.cuda.is_available():
                torch.cuda.synchronize()

    if torch.cuda.is_available():
        torch.cuda.reset_peak_memory_stats()

    monitor = SysMonitor(interval=0.05)
    monitor.start()
    t0 = time.time()
    with torch.no_grad():
        outputs = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False, use_cache=True)
        if torch.cuda.is_available():
            torch.cuda.synchronize()
    elapsed = time.time() - t0
    monitor.stop()
    series = monitor.collect()

    total_tokens = outputs.shape[-1]
    new_tokens = max(total_tokens - input_tokens, 0)
    gpu_peak_gb = (torch.cuda.max_memory_allocated() / 1e9) if torch.cuda.is_available() else 0.0

    return {
        "sec": elapsed,
        "input_tokens": input_tokens,
        "new_tokens": new_tokens,
        "tok_per_s": (new_tokens / elapsed) if elapsed > 0 else float("nan"),
        "gpu_gb_peak": gpu_peak_gb,
        "series": series,
        "output": tokenizer.decode(outputs[0], skip_special_tokens=True)[:500],
        "rss_peak_bytes": summarize_series(series, "rss"),
        "uss_peak_bytes": summarize_series(series, "uss"),
        "swap_peak_bytes": summarize_series(series, "swap_used"),
        "mapped_peak_bytes": summarize_series(series, "smaps_file_pss_bytes") or summarize_series(series, "file_backed_bytes"),
    }


records = []
for label, (tokenizer, model) in [("LFM", (lfm_tok, lfm_model)), ("Qwen", (qwen_tok, qwen_model))]:
    print(f"=== {label} benchmark (device={COMPUTE_DEVICE}, dtype={DTYPE}) ===")
    for target_len in PROMPT_LENGTHS:
        prompt_text = build_prompt_text(target_len)
        result = generate_benchmark(
            model,
            tokenizer,
            prompt_text,
            max_new_tokens=MAX_NEW_TOKENS,
            warmup_new_tokens=WARMUP_NEW_TOKENS,
        )
        records.append({
            "model": label,
            "target_prompt_tokens": target_len,
            "input_tokens": result["input_tokens"],
            "new_tokens": result["new_tokens"],
            "tok_per_s": result["tok_per_s"],
            "wall_time_sec": result["sec"],
            "rss_peak_gb": (result["rss_peak_bytes"] or 0) / (1024 ** 3),
            "uss_peak_gb": (result["uss_peak_bytes"] or 0) / (1024 ** 3),
            "swap_peak_gb": (result["swap_peak_bytes"] or 0) / (1024 ** 3),
            "mapped_peak_gb": (result["mapped_peak_bytes"] or 0) / (1024 ** 3),
        })
        plot_mem(result["series"], title=f"{label} prompt≈{target_len} tokens")
        print(
            f"{label} | target={target_len} | actual_in={result['input_tokens']} | "
            f"tok/s={result['tok_per_s']:.2f} | wall={result['sec']:.2f}s"
        )
        if torch.cuda.is_available():
            print(f"  GPU peak memory: {result['gpu_gb_peak']:.3f} GB")

summary_df = pd.DataFrame(records)
display(summary_df)

from datasets import load_dataset
import torch, math

def calc_ppl(model, tokenizer, texts, max_length=1024):
    model.eval()
    losses = []
    for txt in texts:
        if not txt.strip(): continue
        enc = tokenizer(txt, return_tensors="pt", truncation=True, max_length=max_length).to(model.device)
        with torch.no_grad():
            out = model(**enc, labels=enc["input_ids"])
            loss = out.loss.item()
        losses.append(loss)
    return math.exp(sum(losses)/len(losses)) if losses else float("inf")

ds = load_dataset("wikitext", "wikitext-2-raw-v1", split="test[:200]")
texts = [t for t in ds["text"] if t and not t.isspace()][:64]

print("Calculating Perplexity... (subset)")
lfm_ppl  = calc_ppl(lfm_model, lfm_tok, texts)
qwen_ppl = calc_ppl(qwen_model, qwen_tok, texts)
print("LFM PPL :", lfm_ppl)
print("Qwen PPL:", qwen_ppl)