ResearchForge Documentation

  ┌─────────────────────────────────────────────────────────────┐
  │                    ResearchForge Loop                       │
  │                                                             │
  │  1. SEARCH     arXiv → ranked papers → local knowledge base │
  │       ↓                                                     │
  │  2. HYPOTHESES papers + domain → testable hypotheses        │
  │       ↓                                                     │
  │  3. BASELINE   freeze current metric cryptographically      │
  │       ↓                                                     │
  │  4. RUN        git worktrees × parallel subagents           │
  │                each variant isolated, baseline untouched    │
  │       ↓                                                     │
  │  5. VALIDATE   re-run winner N times → confirm stability    │
  │       ↓                                                     │
  │  6. SHIP       clean branch + report + audit trail          │
  └─────────────────────────────────────────────────────────────┘

pip install "researchforge[serve]"

# After pip install, register skills/rules:
researchforge all install --user

# Claude Code only:
researchforge claude install

# Cursor only:
researchforge cursor install

git clone https://github.com/forger-labs-hq/researchforge
cd researchforge
pip install -e ".[serve,dev]"

docker run --rm -v "$PWD":/workspace -w /workspace \
  ghcr.io/forger-labs-hq/researchforge:latest \
  researchforge research search "your query"

# Recommended — start in the IDE
/researchforge-start
# or
@researchforge-start

# 1. Install
pip install "researchforge[serve]"

# 2. Open Claude Code or Cursor
#    Type one of these:
/researchforge-start
# or
@researchforge-start

# 3. Approve the contract, run the baseline, and let the agent do the rest

pip install "researchforge[serve]"
researchforge all install --user   # → ~/.claude/skills/ and ~/.cursor/rules/
researchforge all status

"""
Your benchmark script. Lives in a protected path.
ResearchForge runs this to measure the baseline, then runs it again
inside each experiment worktree (with the AI's patch applied to src/).
"""
import json
import pathlib
from my_model import load_and_eval  # ← AI can patch this

# Run your evaluation
accuracy, p95_ms, cost = load_and_eval(dataset="benchmark-v2")

# Write results in the standard ResearchForge format
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "accuracy", "value": accuracy},
    "secondary_metrics": {
        "p95_latency_ms": p95_ms,
        "average_cost_usd": cost,
    },
    "sample_count": 1200,
    "seed": 42,
    "metadata": {"dataset_version": "benchmark-v2"},
}), encoding="utf-8")

print("evaluation complete")

{
  "schema_version": 1,
  "primary_metric": {"name": "accuracy", "value": 0.891},
  "secondary_metrics": {
    "p95_latency_ms": 143.2,
    "average_cost_usd": 0.0031,
    "f1_macro": 0.877
  },
  "sample_count": 1200,
  "seed": 42,
  "metadata": {"dataset_version": "benchmark-v2", "model_params": 7340032}
}

objective:
  description: >
    Improve accuracy on the classification benchmark while keeping
    p95 latency under 200ms and cost under $0.005 per query.
  primary_metric:
    name: accuracy
    direction: maximize
  hard_constraints:
    - name: p95_latency_ms
      operator: <=
      value: 200
    - name: average_cost_usd
      operator: <=
      value: 0.005

execution:
  screening_command: python benchmarks/evaluate.py --subset screening
  full_command:      python benchmarks/evaluate.py --subset full
  result_file: artifacts/results.json

import sys
import json, pathlib

subset = "screening" if "--subset" in sys.argv and "screening" in sys.argv else "full"
# screening = fast 10% sample; full = complete eval
dataset_size = 120 if subset == "screening" else 1200

accuracy = run_eval(n_samples=dataset_size)

pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "accuracy", "value": accuracy},
    "sample_count": dataset_size,
    "seed": 42,
}), encoding="utf-8")

researchforge research search "your query" [flags]

# List stored papers
researchforge papers list

# Show details for a specific paper
researchforge papers show paper-003

# Export all papers (for air-gap transfer)
researchforge papers export papers.json

# Import papers from export file
researchforge papers import papers.json

# Delete a paper
researchforge papers delete paper-007

researchforge baseline run [flags]

# Check current baseline status
researchforge baseline status

# Reset (requires explicit confirmation)
researchforge baseline reset --confirm

# Generate hypotheses from stored papers
researchforge hypotheses generate [--n 10] [--model claude-sonnet]

# Interactive review: approve/reject/edit each hypothesis
researchforge hypotheses review

# List all hypotheses and their status
researchforge hypotheses list

# Show details for one hypothesis
researchforge hypotheses show hyp-002

# Manually add a hypothesis
researchforge hypotheses add \
  --title "Per-well normalization" \
  --description "Normalize each well's features independently before training" \
  --evidence "Expert domain knowledge, paper-007"

# Approve/reject without interactive review
researchforge hypotheses approve hyp-001 hyp-003
researchforge hypotheses reject hyp-005 --reason "compute cost too high"

# Auto-generate experiment plan from approved hypotheses
researchforge plan generate

# Import a hand-written plan.yaml
researchforge plan import plan.yaml

# View current plan
researchforge plan show

# Validate plan (check for conflicts, missing scripts, etc.)
researchforge plan validate

researchforge run [flags]

researchforge validate [flags]

researchforge ship [flags]

# Start local hub server
researchforge hub start [--port 8080]

# Hub status
researchforge hub status

# List all experiments across team (requires Hub API key)
researchforge hub experiments --workload nlp-v2

# Approve queued experiments (team lead)
researchforge hub approve exp-012 exp-013

# Install both Claude Code skills and Cursor rules
researchforge all install [--user] [--global]

# --user: installs to ~/.claude/skills/ and ~/.cursor/rules/
# --global: installs to system-wide config (requires admin)

# Verify installation
researchforge all status

# ResearchForge project configuration — full reference
version: "1"

# ── Execution contract ─────────────────────────────────────────
execution:
  setup_command: python -m pip install -e .
  screening_command: python benchmarks/evaluate.py --subset screening
  full_command:      python benchmarks/evaluate.py --subset full
  result_file: artifacts/results.json   # ← your eval script writes here
  timeout_minutes: 20
  max_experiments: 8

# ── Run loop ─────────────────────────────────────────────────────
run:
  stall: 3                  # stop after N non-improvements (optional)
  parallel: 4               # max concurrent subagents
  threshold: 0.005          # minimum Δ to count as improvement (0.5%)

# ── Executor ─────────────────────────────────────────────────────
executor:
  type: venv                # venv | docker | remote
  # docker options:
  image: null               # e.g. "python:3.12-slim"
  build_context: null       # path to Dockerfile context

# ── Permissions ──────────────────────────────────────────────────
permissions:
  editable_paths:
    - src/            # AI can patch these
    - config/
  protected_paths:
    - benchmarks/     # eval script — AI cannot touch
    - evaluator/
    - tests/

# ── Paper search ─────────────────────────────────────────────────
search:
  categories: []            # arXiv category filter
  min_score: 0.60
  max_papers: 20
  rerank: true

# ── Enterprise Hub ───────────────────────────────────────────────
hub:
  url: ${RESEARCHFORGE_HUB_URL}
  api_key: ${RESEARCHFORGE_API_KEY}
  workload: null            # tag all runs under this workload name
  require_approval: false   # queue experiments for team lead approval

version: "1"
baseline_commit: abc1234
baseline_metric: 15.2441

experiments:
  - id: exp-001
    hypothesis: hyp-005
    description: Per-well normalization before feature engineering
    env:
      NORMALIZE_PER_WELL: "true"
      SCALER: standard

  - id: exp-002
    hypothesis: hyp-006
    description: Multi-scale rolling window features
    env:
      WINDOW_SIZES: "5,20,50"
      FEATURE_TYPE: rolling

  - id: exp-003
    hypothesis: hyp-001
    description: GRU encoder with depth positional encoding
    env:
      MODEL_TYPE: gru
      POSITIONAL_ENCODING: depth_normalized
      HIDDEN_SIZE: "128"
    # This experiment builds on exp-001 (run exp-001 first)
    depends_on: exp-001

  - id: exp-004
    hypothesis: hyp-003
    description: Multi-task aux prediction
    env:
      AUX_TARGETS: "gamma,resistivity"
      AUX_WEIGHT: "0.3"
    # Only run if exp-003 passed
    requires_pass: exp-003

# Required for enterprise features only
RESEARCHFORGE_HUB_URL=https://hub.yourcompany.com
RESEARCHFORGE_API_KEY=rf_live_xxxxxxxxxxxx

# Optional: override default model for hypothesis generation
RESEARCHFORGE_LLM=claude-sonnet-4-5   # default
# RESEARCHFORGE_LLM=gpt-4.1
# RESEARCHFORGE_LLM=http://localhost:11434/api (Ollama)

# Air-gap mode: disable all external calls
RF_OFFLINE=false
RF_ARXIV_DISABLED=false

# Logging
RF_LOG_LEVEL=info          # debug | info | warn | error

import os
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
import numpy as np

# ResearchForge injects these via worktree env vars
model_type = os.environ.get("MODEL_TYPE", "gbm")
n_estimators = int(os.environ.get("N_ESTIMATORS", "100"))
normalize = os.environ.get("NORMALIZE", "false").lower() == "true"

X_train, X_val, y_train, y_val = load_data()

if normalize:
    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_val = scaler.transform(X_val)

if model_type == "rf":
    model = RandomForestRegressor(n_estimators=n_estimators, random_state=42)
else:
    model = GradientBoostingRegressor(n_estimators=n_estimators, random_state=42)

model.fit(X_train, y_train)
preds = model.predict(X_val)
rmse = np.sqrt(mean_squared_error(y_val, preds))

# Write results.json — NOT print(RF_METRIC)
import json, pathlib
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "rmse", "value": float(rmse)},
    "sample_count": len(y_val),
    "seed": 42,
}), encoding="utf-8")

experiments:
  - id: exp-001
    env: { MODEL_TYPE: gbm, N_ESTIMATORS: "200" }
  - id: exp-002
    env: { MODEL_TYPE: rf, N_ESTIMATORS: "200" }
  - id: exp-003
    env: { MODEL_TYPE: gbm, N_ESTIMATORS: "200", NORMALIZE: "true" }

import os
import torch
import pytorch_lightning as pl

lr = float(os.environ.get("LR", "1e-3"))
hidden = int(os.environ.get("HIDDEN_SIZE", "256"))
dropout = float(os.environ.get("DROPOUT", "0.1"))
use_batchnorm = os.environ.get("BATCHNORM", "false") == "true"

class MyModel(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.net = build_net(hidden, dropout, use_batchnorm)
        self.lr = lr

    def training_step(self, batch, idx):
        loss = self.net(batch)
        return loss

    def validation_step(self, batch, idx):
        val_loss = self.net(batch)
        # Emit to ResearchForge
        self.log("rf_val_loss", val_loss)
        return val_loss

trainer.fit(model, train_dl, val_dl)

# Write results.json from best checkpoint metrics
import json, pathlib
best_val = trainer.callback_metrics.get("val_loss", float("inf"))
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "val_loss", "value": float(best_val)},
    "sample_count": len(val_dl.dataset),
    "seed": 42,
}), encoding="utf-8")

import os
from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer
from datasets import load_dataset
import numpy as np

model_name = os.environ.get("MODEL_NAME", "distilbert-base-uncased")
lr = float(os.environ.get("LR", "2e-5"))
epochs = int(os.environ.get("EPOCHS", "3"))
warmup = float(os.environ.get("WARMUP_RATIO", "0.1"))

model = AutoModelForSequenceClassification.from_pretrained(model_name)

args = TrainingArguments(
    output_dir="./out",
    learning_rate=lr,
    num_train_epochs=epochs,
    warmup_ratio=warmup,
    evaluation_strategy="epoch",
    save_strategy="no",
    load_best_model_at_end=False,
    report_to="none",  # disable wandb/mlflow — RF handles tracking
)

trainer = Trainer(model=model, args=args, ...)
trainer.train()
results = trainer.evaluate()

# Write results.json
import json, pathlib
pathlib.Path("artifacts").mkdir(exist_ok=True)
pathlib.Path("artifacts/results.json").write_text(json.dumps({
    "schema_version": 1,
    "primary_metric": {"name": "f1", "value": results["eval_f1"]},
    "secondary_metrics": {"eval_loss": results["eval_loss"]},
    "sample_count": len(eval_dataset),
    "seed": 42,
}), encoding="utf-8")

# ResearchForge uses your active venv as the template
# Activate your env, then run baseline:
source .venv/bin/activate
researchforge baseline run

# Worktrees are created at:
# .rf-worktrees/exp-001/venv/   ← isolated copy
# .rf-worktrees/exp-001/repo/   ← git worktree at baseline commit

# To add extra deps for a specific experiment, use the plan:
#   experiments:
#     - id: exp-001
#       pip_install: ["torch-geometric==2.5.0"]

executor:
  type: docker
  image: python:3.12-slim    # or your custom image
  build_context: .           # uses your Dockerfile if present

# Or override at runtime:
researchforge run --executor docker --image my-ml-image:latest

name: ResearchForge experiments

on:
  workflow_dispatch:
  push:
    branches: [research/**]

jobs:
  run-experiments:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.12" }

      - name: Install ResearchForge
        run: pip install "researchforge[serve]"

      - name: Install project deps
        run: pip install -r requirements.txt

      - name: Freeze baseline
        run: researchforge baseline run

      - name: Run experiments
        run: researchforge run --stall 3 --parallel 2

# Start the local monitoring server for this project
researchforge serve --background

# Start the machine-wide hub dashboard
researchforge hub --background

# Inspect project state and monitor status
researchforge status
researchforge paths

# Stop after 3 consecutive non-improvements
researchforge run --stall 3 --threshold 0.005

Orchestrator
├─ subagent: exp-001  (.rf-worktrees/exp-001/)
│    env: MODEL_TYPE=gru, LR=1e-3
│    runs: python benchmarks/evaluate.py --subset full
│    writes: artifacts/results.json → accuracy=0.891
│    reports back to orchestrator
│    cleans up: ✓
│
├─ subagent: exp-002  (.rf-worktrees/exp-002/)  ← runs in parallel
│    env: MODEL_TYPE=transformer, LR=1e-3
│    ...
│
└─ subagent: exp-003  (.rf-worktrees/exp-003/)  ← runs in parallel
     env: MODEL_TYPE=gru, LR=3e-4
     ...

protected:
  - config/prod.yaml     # production config must not change
  - src/api/             # API surface must not change
  - tests/               # test suite must not be modified
  - data/raw/            # raw data must not be touched

# Check which files are currently protected
researchforge protected list

# Verify no experiments have pending violations
researchforge protected verify

# View audit log
researchforge audit log [--last 20]

# Export full audit log as JSON
researchforge audit export audit.json

# Verify log integrity (detects tampering)
researchforge audit verify

{"ts":"2026-08-06T09:14:22Z","event":"baseline.frozen",
 "commit":"abc1234","metric":"rmse","value":15.2441,
 "user":"manas@forger-labs.com","hmac":"a3f7..."}

{"ts":"2026-08-06T09:31:05Z","event":"experiment.completed",
 "id":"exp-003","status":"pass","metric":"rmse","value":11.44,
 "delta":3.80,"stall_count":0,"worktree":".rf-worktrees/exp-003"}

# Pull and start the hub
docker pull ghcr.io/forger-labs-hq/researchforge-hub:latest

docker run -d \
  --name rf-hub \
  -p 8080:8080 \
  -v /data/rf-hub:/data \
  -e RF_SECRET_KEY=$(openssl rand -hex 32) \
  -e RF_ADMIN_EMAIL=admin@yourcompany.com \
  ghcr.io/forger-labs-hq/researchforge-hub:latest

# Dashboard available at http://your-server:8080

RESEARCHFORGE_HUB_URL=https://hub.yourcompany.com
RESEARCHFORGE_API_KEY=rf_live_xxxxxxxxxxxx

RF_HUB_URL=https://hub.yourcompany.com \
RF_API_KEY=rf_live_xxxx \
researchforge run --workload search-ranking-v3

name: RF experiments on PR

on:
  pull_request:
    paths: ["src/**", "experiments/**"]

jobs:
  experiments:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: pip install "researchforge[serve]"
      - run: pip install -r requirements.txt
      - run: |
          researchforge baseline run
          researchforge run --stall 3 --workload "pr-${{ github.event.number }}"
        env:
          RESEARCHFORGE_HUB_URL: ${{ secrets.RF_HUB_URL }}
          RESEARCHFORGE_API_KEY: ${{ secrets.RF_API_KEY }}
      - run: |
          # Post experiment results as PR comment
          researchforge hub comment \
            --pr ${{ github.event.number }} \
            --repo ${{ github.repository }}

# 1. Export hub image on connected machine
docker save ghcr.io/forger-labs-hq/researchforge-hub:latest \
  | gzip > rf-hub.tar.gz

# 2. Export pip wheels on connected machine
pip download "researchforge[serve]" -d ./rf-wheels/
tar -czf rf-wheels.tar.gz rf-wheels/

# 3. Transfer both tarballs to air-gapped machine, then:
docker load < rf-hub.tar.gz
tar -xzf rf-wheels.tar.gz
pip install --no-index --find-links=./rf-wheels "researchforge[serve]"

# 4. Pre-populate paper cache from connected machine:
researchforge papers export papers.json   # on connected machine
# Transfer papers.json, then:
researchforge papers import papers.json   # on air-gapped machine

# 5. Disable external calls
export RF_OFFLINE=true
export RF_ARXIV_DISABLED=true

# Machine A — researcher 1
RESEARCHFORGE_HUB_URL=https://hub.internal
RESEARCHFORGE_API_KEY=rf_live_xxxx
researchforge run --worker --tags gpu-a100 --parallel 4

# Machine B — researcher 2 (picks up remaining experiments from queue)
RESEARCHFORGE_HUB_URL=https://hub.internal
RESEARCHFORGE_API_KEY=rf_live_yyyy
researchforge run --worker --tags gpu-rtx6000 --parallel 2

.researchforge/
├── config.yaml          ← researchforge.yaml (symlink)
├── baseline.json        ← frozen baseline record + HMAC
├── papers/              ← local knowledge base (JSON)
├── hypotheses.yaml      ← generated + reviewed hypotheses
├── plan.yaml            ← current experiment plan
├── experiments/
│   ├── exp-001.json     ← per-experiment result record
│   ├── exp-002.json
│   └── ...
├── reports/
│   └── exp-009-final.json  ← engineering report from ship
├── audit.log            ← append-only audit trail (JSON Lines)
└── lineage.json         ← full experiment DAG

# Upgrade to latest
pip install --upgrade "researchforge[serve]"

# Re-install IDE integrations after upgrade
researchforge all install --user

# Check version
researchforge --version

# Migrate project state to new format (if breaking change)
researchforge migrate --dry-run  # preview changes
researchforge migrate            # apply