Qwen 3.5 로컬 설치 완전 가이드 — Ollama부터 vLLM까지

curl -fsSL https://ollama.com/install.sh | sh

brew install ollama

ollama --version

# 9B 모델 (8GB GPU 추천)
ollama run qwen3.5:9b

# 35B MoE 모델 (24GB GPU 추천, 스위트스팟)
ollama run qwen3.5:35b-a3b

# 4B 모델 (가벼운 용도)
ollama run qwen3.5:4b

# 0.8B 모델 (CPU 전용도 OK)
ollama run qwen3.5:0.8b

>>> 파이썬으로 피보나치 수열 생성기를 만들어줘

# 모델을 백그라운드로 로드
ollama serve &

# API 호출
curl http://localhost:11434/api/generate -d '{
  "model": "qwen3.5:9b",
  "prompt": "Python으로 REST API 서버를 만드는 코드를 작성해줘",
  "stream": false
}'

curl http://localhost:11434/v1/chat/completions -d '{
  "model": "qwen3.5:9b",
  "messages": [{"role": "user", "content": "Hello!"}]
}'

git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
cmake -B build -DGGML_CUDA=ON
cmake --build build --config Release -j

cmake -B build
cmake --build build --config Release -j

# huggingface-cli 설치
pip install huggingface_hub

# 35B-A3B Q4_K_M 다운로드 (스위트스팟)
huggingface-cli download bartowski/Qwen_Qwen3.5-35B-A3B-GGUF \
  --include "Qwen3.5-35B-A3B-Q4_K_M.gguf" \
  --local-dir ./models

./build/bin/llama-cli \
  -m ./models/Qwen3.5-35B-A3B-Q4_K_M.gguf \
  -c 8192 \
  -ngl 99 \
  --temp 0.7 \
  -p "You are a helpful assistant."

./build/bin/llama-server \
  -m ./models/Qwen3.5-35B-A3B-Q4_K_M.gguf \
  -c 8192 \
  -ngl 99 \
  --host 0.0.0.0 \
  --port 8080

pip install vllm

vllm serve Qwen/Qwen3.5-35B-A3B \
  --dtype auto \
  --max-model-len 32768 \
  --tensor-parallel-size 1 \
  --gpu-memory-utilization 0.9

vllm serve Qwen/Qwen3.5-122B-A10B \
  --dtype auto \
  --max-model-len 32768 \
  --tensor-parallel-size 2 \
  --gpu-memory-utilization 0.9

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed"
)

response = client.chat.completions.create(
    model="Qwen/Qwen3.5-35B-A3B",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain quantum computing in simple terms."}
    ],
    temperature=0.7,
    max_tokens=1024
)

print(response.choices[0].message.content)

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

model_name = "Qwen/Qwen3.5-9B"

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "파이썬으로 웹 스크래퍼를 만들어줘"}
]

text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(text, return_tensors="pt").to(model.device)

outputs = model.generate(
    **inputs,
    max_new_tokens=1024,
    temperature=0.7,
    do_sample=True
)

response = tokenizer.decode(outputs[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)
print(response)

model = AutoModelForCausalLM.from_pretrained(
    "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

ollama run qwen3.5:9b
>>> /set parameter num_ctx 32768
>>> Think step by step: 다음 수학 문제를 풀어줘.
>>> 1부터 100까지 자연수 중 3의 배수이면서 5의 배수가 아닌 수의 합은?

messages = [
    {"role": "system", "content": "You are a helpful assistant. Think step by step before answering."},
    {"role": "user", "content": "Prove that the square root of 2 is irrational."}
]

text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
    enable_thinking=True  # Thinking 모드 활성화
)

from transformers import AutoProcessor, AutoModelForCausalLM
import torch

model_name = "Qwen/Qwen3.5-9B"
processor = AutoProcessor.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://example.com/chart.png"},
            {"type": "text", "text": "이 차트를 분석해줘. 주요 트렌드는 뭐야?"}
        ]
    }
]

inputs = processor.apply_chat_template(messages, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=1024)
print(processor.decode(outputs[0], skip_special_tokens=True))

# Ollama: 컨텍스트 줄이기
ollama run qwen3.5:35b-a3b
>>> /set parameter num_ctx 4096

# llama.cpp: Flash Attention 활성화
./build/bin/llama-cli -m model.gguf -c 8192 -ngl 99 -fa

# 40개 레이어만 GPU에 올리기 (나머지는 CPU)
./build/bin/llama-cli -m model.gguf -c 4096 -ngl 40

# llama.cpp: KV 캐시를 Q8로 양자화
./build/bin/llama-cli -m model.gguf -c 32768 -ngl 99 -ctk q8_0 -ctv q8_0