Repository navigation
Expand file tree
/
Copy pathdistributed_engine.py
More file actions
248 lines (220 loc) · 9.49 KB
/
Copy pathdistributed_engine.py
File metadata and controls
248 lines (220 loc) · 9.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
"""Resident inference: rank 0 owns conditioning/decoders, all ranks sample.
Launch with torchrun --standalone --nproc_per_node=8. No external task submission.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from datetime import timedelta
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
for path in (ROOT, ROOT / "src", ROOT / "scripts" / "eval"):
if str(path) not in sys.path:
sys.path.insert(0, str(path))
import torch
import torch.distributed as dist
from scripts.demo.resident_engine import Engine, _atomic_json
from stage2.models import ulysses
class SamplerGroup:
def __init__(self, model, device):
self.model, self.device = model, device
self.busy = False
# Idle workers wait on CPU; an idle NCCL broadcast would trigger its
# watchdog and take down an otherwise healthy resident service.
self.control = dist.new_group(backend="gloo", timeout=timedelta(hours=48))
def _broadcast(self, values=None):
meta = None
if dist.get_rank() == 0:
tensors = {k: v for k, v in values.items() if torch.is_tensor(v)}
meta = dict(
options={k: v for k, v in values.items() if k not in tensors},
tensors={
k: [list(v.shape), str(v.dtype).split(".")[-1]] for k, v in tensors.items()
},
)
messages = [meta]
dist.broadcast_object_list(messages, src=0, group=self.control)
meta = messages[0]
if meta["options"].get("stop"):
return meta["options"]
result = dict(meta["options"])
for key, (shape, dtype) in meta["tensors"].items():
tensor = (
values[key].contiguous()
if dist.get_rank() == 0
else torch.empty(shape, dtype=getattr(torch, dtype), device=self.device)
)
dist.broadcast(tensor, src=0)
result[key] = tensor
return result
def __call__(self, model, z_ref, views, cond_num, **kwargs):
from eval_generation import sample_v4_euler
values = dict(kwargs, z_ref_clean=z_ref, total_view=views, cond_num=cond_num)
# Workers raise on the same checks and a worker exception stops the
# whole service, so reject unsupported options before broadcasting.
if ulysses.cfg_parallel_enabled() and values.get("guidance_mode", "cfg") not in ("cfg", "none", "ig"):
raise ValueError("CFG parallel supports cfg/ig/none guidance only")
self.busy = True
self._broadcast(values)
result = sample_v4_euler(model, **values)
self.busy = False
return result
def configure(self, *, fold_text_queries=None, attention_backend=None):
if self.busy:
raise RuntimeError("Cannot change inference options during sampling")
options = {}
if fold_text_queries is not None:
options["fold_text_queries"] = bool(fold_text_queries)
if attention_backend is not None:
options["attention_backend"] = attention_backend
self._validate_configuration(options)
self._broadcast({"configure": options})
self._apply_configuration(options)
@staticmethod
def _validate_configuration(options):
if set(options) - {"fold_text_queries", "attention_backend"}:
raise ValueError("Unsupported inference configuration")
if "fold_text_queries" in options and type(options["fold_text_queries"]) is not bool:
raise ValueError("fold_text_queries must be boolean")
if options.get("attention_backend", "auto") not in {"auto", "fa3"}:
raise ValueError("Unsupported attention backend")
def _apply_configuration(self, options):
self._validate_configuration(options)
for block in self.model.blocks:
if "fold_text_queries" in options:
block.fold_text_queries = options["fold_text_queries"]
if "attention_backend" in options:
block.attn.attention_backend = options["attention_backend"]
def serve_workers(self):
from eval_generation import sample_v4_euler
while True:
values = self._broadcast()
if values.get("stop"):
return
if "configure" in values:
self._apply_configuration(values["configure"])
continue
if "device" in values:
values["device"] = self.device
with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
sample_v4_euler(self.model, **values)
def stop(self):
if self.busy:
raise RuntimeError(
"Cannot stop through metadata while a sampler collective is incomplete"
)
self._broadcast(dict(stop=True))
def setup(checkpoint_dir, cfg_parallel):
local = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(local)
ulysses.init_ulysses(cfg_parallel=cfg_parallel)
device = torch.device("cuda", local)
if dist.get_rank() == 0:
engine = Engine(checkpoint_dir, device)
model = engine.model.flow
else:
from gae.pipeline import load_flow
model = load_flow(
str(ROOT / "configs/flow_gae64.yaml"),
str(Path(checkpoint_dir) / "flow_gae64.pt"),
device,
)
model.fast_inference = True
engine = None
group = SamplerGroup(model, device)
if engine is not None:
engine.sampler = group
dist.barrier(device_ids=[local])
return engine, group
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--serve", action="store_true")
p.add_argument("--host", default="127.0.0.1")
p.add_argument("--port", type=int, default=7860)
p.add_argument("--checkpoint-dir", required=True)
p.add_argument("--cfg-parallel", action="store_true")
p.add_argument("--image", required=True)
p.add_argument("--prompt-file", required=True)
p.add_argument("--poses")
p.add_argument("--views", type=int, default=81)
p.add_argument("--steps", type=int, default=25)
p.add_argument("--warmup-steps", type=int, default=2)
p.add_argument("--repeat", type=int, default=1)
p.add_argument("--seed", type=int, default=42)
p.add_argument("--save-parity", action="store_true")
p.add_argument("--output", type=Path, required=True)
a = p.parse_args()
engine, group = setup(a.checkpoint_dir, a.cfg_parallel)
if dist.get_rank() != 0:
group.serve_workers()
else:
if a.serve and os.environ.get("GAE_CAPTION", "1") != "0":
import threading
# transformers resolves submodules lazily and concurrent first
# imports from two threads can see a half-initialized module, so
# import on this thread before loading in the background.
from transformers import ( # noqa: F401
AutoProcessor,
Qwen2_5_VLForConditionalGeneration,
TextIteratorStreamer,
)
from scripts.demo import captioner
# Loading the caption model takes about a minute; overlap it with warm-up.
threading.Thread(target=captioner.warm, daemon=True).start()
a.output.mkdir(parents=True, exist_ok=False)
report = {
"status": "running",
"world_size": dist.get_world_size(),
"cfg_parallel": a.cfg_parallel,
"requests": [],
}
_atomic_json(a.output / "benchmark.json", report)
common = dict(
image=a.image,
prompt=Path(a.prompt_file).read_text().strip(),
poses=a.poses,
views=a.views,
seed=a.seed,
stride=2,
)
try:
if a.warmup_steps:
report["warmup"] = engine.generate(
**common, steps=a.warmup_steps, output_dir=a.output / "warmup"
)
_atomic_json(a.output / "benchmark.json", report)
if a.serve:
import camera_studio as app
app.RESIDENT_ENGINE = engine
report["status"] = "ready"
_atomic_json(a.output / "benchmark.json", report)
app.demo.queue(default_concurrency_limit=1).launch(
server_name=a.host,
server_port=a.port,
share=os.environ.get("GRADIO_SHARE", "0") == "1", css=app.CSS,
allowed_paths=[str(app.OUTPUT_ROOT), str(ROOT / "examples"),
str(ROOT / "scripts/demo/camera_editor.js"),
str(ROOT / "scripts/demo/camera_studio.css")],
)
for i in range(0 if a.serve else a.repeat):
result = engine.generate(
**common,
steps=a.steps,
output_dir=a.output / f"request-{i:03d}",
save_parity=a.save_parity,
)
report["requests"].append(result)
_atomic_json(a.output / "benchmark.json", report)
print(json.dumps(result), flush=True)
report["status"] = "complete"
except Exception as exc:
report.update(status="failed", error=f"{type(exc).__name__}: {exc}")
raise
finally:
_atomic_json(a.output / "benchmark.json", report)
if not group.busy:
group.stop()
dist.destroy_process_group()
if __name__ == "__main__":
main()