From 6decf56b2d604d80c96b16faf32ef74f8cac672a Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Wed, 15 Apr 2026 11:05:25 +0200 Subject: [PATCH] refactor(tooling): drop --shard from gemma_naming.py (#833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallelism via two concurrent sr-voice subprocesses does not work on this ROCm + llama-cpp-rs setup — launching a second instance poisons the first one's GPU context (both fall back to 0% GPU / 50% CPU busy-loop and stop making progress). Verified empirically: single shard runs cleanly at ~1.2s/feature, two shards deadlock. Without a working parallel path, --shard is dead weight. Resume semantics were already free: the pipeline skips bodies whose markers.json has non-empty name fields (preserved path), so a killed run re-starts just by re-running the same command. Simplifications: - Remove --shard argument and all slicing logic. - Remove banner_shard / shard_offset / shard_n / shard_m plumbing. - Rename internal total_shard_systems → total_systems. - Default --log path is now .tmp/gemma_naming.log (was conditional on --shard). Pass `--log -` to disable file logging. - Startup banner now prints a one-line resume reminder so the user can see at a glance that a killed run is recoverable. --- server/data/systems.db | Bin 2150400 -> 2150400 bytes tooling/planet-gen/gemma_naming.py | 82 +++++++---------------------- 2 files changed, 19 insertions(+), 63 deletions(-) diff --git a/server/data/systems.db b/server/data/systems.db index fe2ea269626e7b9a4e308976e1f11447268475db..8ca10c47d8386016ac522933a8555cd34e68e608 100644 GIT binary patch delta 4504 zcmZu#d2kcg8Q(*;o-ZzA8(Ubrin&}@V@Z|-Hn;(dZG4*}5E2q2B-yO3%aV+NLJf`x z|0F~^naP{B)4~+UaCH){tii;l&`#-;kdP+a5JCV$NWff~CLP+or`2kewHS8h_x--_ z_ul*7`@Xj;Z`}R{xe-4hGDL;wkSdfEN)D+*Ae0hH4W)(BLm8n_q0yl+p|PQHq0G?u z!&&`V>e4bpNtw|ioXpC4rj1zx=B(wP7rIku6{VW<&_{M)1m3uQgKWoWtopqVNe-ie zQ&j}%5h~Q#13;Z}i>y&UCt&(;(!0X%$7Ek8>E|FrSU)biJBe>jtsG73s4B1D%U1Zk zfdJVthAt58D$eP01ju$wEVixea@pAVUYCRPjH9Pg6<&|s>vyote$MSAIbrUE>_L)r z3sheA3q?8RR62*MuAOdlkR}P*Nz0a*mD%XYLC*wXW@YYHPmOylJz0z_wsXO?Ehe%< z!5GPQ1!EvNiLr|+1Il*X;oKW13(R=6k(rBqUarMJnk2#=B`iW#cV5+shQ~M!F|ykh zboyJY)FUNjq=qHRUMxmUr=BV)BlSeseIc(rv!_!^)&YDn^vcj9L$?fFGVG9Hy9_xQ zu99J!3|k{`5!!J(SS!(XuttX6GF&agE*S=87?7b~5bo#oOzU|pQ7geC84kbXp^xt=rS&?%IT?N?!%t;+R)(L*@QeiQ z;0GB#i@^-=G={+wVNk0H{6Zf)K?fB(K?fB(K|3XDP(8QQXof3#Iy33QXpi{0)u=~| z2H=y*-%%`brNFDCW`M{gA$FG-H66H>QC8qmQWnq=NA*PJG=ugyE<%+6PDz=-DkWtE zZA!`jS|!RJUH){iR!LdGnh4c{`zab`0o}?lGgz&pN1`krqIf{<9RZL_ntl$SFRRW$VDHDi$pF|H8Jb}6kn#}S?$|Ez) z0K+4=>EPiA&IBI7PBcW}_ez(s6nr;=n*r_%f4-nOwNfv?l}PzjM9QxsGWII6qj&Bf z$|k-(g?>5RFZ_4KM0YaTA-LKmzDS|(f&JXXK1zeWRkA4jUMJg8@#0I+Zu4l-=xit$z4s_!A_b@D?2oH`>Sd=Y|m8*Z=cp~qcmomMGzfSfD4kQc*7!8|5JN8 zmkh!eZfmP!L-}rNkCCJw77c0Zq6t_QZ0>{`hqSely=1cweletNlI%sB15kfgTOS)_ z<6UiUbrL~XiK|d;U{PA2-@aOh`#7o56R z_E&~GVEbO?SJHU1!|m{3oWCN>!DoA!S(1NNcom$vPh316e@?g!uG=TCZ*I614(wxQ z;vL2QH}^9i?PrrzOQ~EU*^qRTYE5rUC(=Ghv!#tm9iT^11F7!Ryp)qE9VvO>6!3vu z^+~lyos+~TRjPhe?Nz;|N~Qbhj^w|M{$R9ibk?Z9j|z^OOyy?$D`QKBHT@o{(T_>G zmiNQc2iPf-=B(iY>>?lMaT5Qgi67}~MYZGegslhJrgR&Tiuw=zYv7l{myWSj(G0E4 z{1W(&gKSN%n1MApBuBQV73Sm@K<*H$E3WTyJ3Z_|eBBXm5!;BZ#J2JTv(@t)7BTPy z5~VEht`V09hYzvYqoWnp%&)>RMwoS&)tQ?8&Q_;A*yVS!^PODVs$gJ!-TL~91Y_6o zZCE!M+L7*;(cdWxIY*oG*UWk}($6~`=U&gZVnrc5j1+|pPCw`1Y$yw?`3=2}*p)z2 z!#m;g!|bGKV#bT?HownBYC1`_IbpN|)*N9c=@&UYwqRF4+#w6FrdSe0H}dwhsX3ZL z_`4%411fBO2j{W5;E5xwM(Rm;^9Z|x-cHOx*JC+o;yZ8xQ{kaOl(O_oOg!A0N}ud+#c?l;<SLDCN4hgoYjwnX3yI&7Ns8rTNfRHy5|N;|i%E(39x;sN=ojsdNmmI=xhZ z;;CFAu1rWUOAYVA5(XYak|KNuhzsjC5PvoL5(w&eHx?AY3{>U%=uLxrl-ONI?5@=$ hkTmixEGdL7NRl2)2TcUu8Dt}WR>8ab{t0%>{{h^|*Z2Sc delta 4624 zcmZu!3s6&68ouY=Kn@Q@!9X=6_oP;lVoGiT;h|I!-RXAk$>ro`FJa~~ z-~av3^Plsdd$a$@^I$)Df*{BSxu7l>7mN?;gD{v7ObjLklY?`DbAu_tdBOR?)L>e0 z!CAwkL0?#$Z!6AsNLLJo@68cuD$lUvFWDUloQ~C%-tPfz1i|OO{0evpny)|fHt-O% z_-Z)=+NCmm`V`c|1)xg*9fOk}#Jwi{c3%3Ifcp+|(!SL6J#qM_jP#V%ZS%OrY`H(+ zN98_mr>9pA_fIB~3I7pQqpNjyrwdj3J3XL1mCItw{M~MUz=P@nzAi8DNNXNSp9Y{q zVi%=9oRc9Wav5wzb%EdkxfHsA?WGm#(&Z;ZejbumtXud>cU8xHZi!-<>-P0M(`*NA z8b$zK4Pyfy%D5FTo4nN2vkSkvvREM1hBN2(_e4G78N$B zaI*>#eJMM;FO1q&)g>DtLs?epv78N$DaHk~QS=7I*|3gZ(!VgsVz6#${ z;kzolp~CAbyr#mdD!ih?%PPDShFS8;TH&-xPpR;t3NJ{1TVVVQ_D?2rMrCiU0-T5z z&PNO5(Sj5$oQoF5qJ^{3!f3QG5)u04tnWl5S$I2Ic#GESfu~e>QiUf}_@)Yvt8kD) zH$0=l(-Ala4o5H?lBUhZ-bbx>Ddl0mRpA{9-Ryl8-jkZEj1S8CpG@PjXJ#be`-MCs zf(`DV$KR8ya7A#tmMVnXv{V6nLQ6T}RxOnew?t9>;hcG}DT)hI4!BuM*O>1tz)!puG7M$?PEG`dTp2a!frCFQ}PSY}IydpTIr5x~L zn3_vQkfwzf@Zyh+g)OJ3wOn(!pdxruOXb58S}G5|sio}jxRw&&ApY~m#x$-FP7+`M z6N>DB=M`Xw<6*GS1|sSn>9iENB3UCx3_{6v%XJi4FqlxE) z@60Ci;M@3;>`?m_Enhw{;Z``|87<|2r?r$F4r?g^4q?lu#9z$E6&`xlnW$p zmHyk3*%c4kq|WxtRjm9yFzm}5XN~fQih(8Xb>Ni`FS!KWt{z{Xt5f#v@5~$+jR%EH z5pxIYk;$)P?ts$Q<{#O`RS($k56D!OCVvD8#dbmVO$dLA)C~xDt;JN8IHRZ;$i>)V zT1&r=Ubkfrugw*aP(5FpCjjWeIk(LLH2)R9;k@B!#J74Sa0Jy;tsQBuuSe+4p%kMhW~Ek*CfrvmOEA=D~h;=fexG_ z@at*Ust3IIfWVj0By|HG{6~Qgca;=XJJ60TcD_84wA;>C?*d+Y{wQBSgOr3k_{XEX zlgg_@Zd`nfFQW3QP%D1!7+*-`H6a(CKE|)0^6F3v)(xumD?`n=Vvx_L`qI!&{Nf;A zOy#^+doVc8r%7K7^8aDMllZSE6eX#6ZD6*ZGCf*izi)~95lKzo&D5)~>hr~DLygcXeb?(+vwqqm#+7svdYOV=EGGANtG>ZI%pQx3LHAxloF zx39w+@VUBCJ-N%6ufR{t7vSedVmRWq5wvl^LMAmlp^<-eVlJLPimDRiC6h^t4`Q&5 zD~F9@8ObBy-ce*pZS3%(CON}y7x^vWbv@mS504^4itI#+su$NNMO~GR`_-btmdr3< znTx;xoiVDa5le|@79fA&d5^ri^2qeJDtEMySL*e5l68hd?yK3izAQ$x4dUvgY%arO z#on`WA$0+N7ikc6`z5~*Sz@@=h^uH33Cg7tDZ&E630y*3#Vpue?rWJM*D4AwPz7 zt+BKi=ODb;= z^RekXs-SYCn1{R1tNP8N17AOnHc)+?Xvd}r6nR^05Cz;bq59W~Hhg&mmD5V<#Yf3@ zS@DetWJ#gNrIs_wv-=qS_5upu(WFZoL@O@3 zh$?8UmEv-ehR3g7MEvq5|6Ww@^R#mW|Exg^4RF-O_lo3h6;`ujb6;&w1M zTbR$|yJC1@+Z!o`T%A~j_e`Os_R;|9hAJQZvr>jn^P&U9cEzc9V8#V`8$y_L2lQ!m>&=bRNqgYP-a!+7qsGAY}88IB|#kESEe>jIM%Oh*6^!nO6 V`g)mPgK_2{7)LQYk6%Lb{s)nT+Hn8? diff --git a/tooling/planet-gen/gemma_naming.py b/tooling/planet-gen/gemma_naming.py index e6b1077ed..0b2f12923 100755 --- a/tooling/planet-gen/gemma_naming.py +++ b/tooling/planet-gen/gemma_naming.py @@ -1185,22 +1185,12 @@ def main(): "'Meridian') may appear across the full run before dedup " "starts rejecting it. 0 = disabled. Default: 20.", ) - parser.add_argument( - "--shard", - default="0/1", - help="Process a slice of the body list for parallel runs. Format " - "N/M: worker N of M total. Each worker walks bodies in hop " - "order, picking every Mth body starting at offset N. Example: " - "run `--shard 0/2` in one terminal and `--shard 1/2` in " - "another to split the work in half. Default: 0/1 (all bodies).", - ) parser.add_argument( "--log", - default=None, + default=str(REPO_ROOT / ".tmp" / "gemma_naming.log"), help="Path to a log file. Every status line is written to both " - "stdout and the log. Defaults to " - ".tmp/gemma_naming.shard{N}of{M}.log when --shard is not " - "trivially 0/1; disabled otherwise. Pass '-' to disable.", + "stdout and the log. Default: .tmp/gemma_naming.log. " + "Pass '-' to disable file logging.", ) parser.add_argument( "--verbose", @@ -1210,31 +1200,7 @@ def main(): ) args = parser.parse_args() - # Parse shard spec. - try: - shard_n_str, shard_m_str = args.shard.split("/", 1) - shard_n = int(shard_n_str) - shard_m = int(shard_m_str) - if shard_m < 1 or not (0 <= shard_n < shard_m): - raise ValueError - except ValueError: - print( - f"error: invalid --shard '{args.shard}'. " - "Expected N/M with 0 <= N < M, e.g. 0/2.", - file=sys.stderr, - ) - sys.exit(1) - - # Default log path: enable for non-trivial shards, disable for 0/1. - if args.log is None: - if shard_m > 1: - log_path = REPO_ROOT / ".tmp" / f"gemma_naming.shard{shard_n}of{shard_m}.log" - else: - log_path = None - elif args.log == "-": - log_path = None - else: - log_path = Path(args.log) + log_path = None if args.log == "-" else Path(args.log) db_path = Path(args.db) if not db_path.exists(): @@ -1278,50 +1244,41 @@ def main(): ensure_atlas_schema(conn) hop_order = load_body_hop_order(conn) - all_markers = discover_bodies(args.body, args.limit, hop_order) - if not all_markers: + markers_paths = discover_bodies(args.body, args.limit, hop_order) + if not markers_paths: log(f"error: no markers.json found (body={args.body})") conn.close() sys.exit(1) - # Apply shard slicing AFTER discovery + sort. Shard 0/2 gets indices - # [0, 2, 4, …], shard 1/2 gets [1, 3, 5, …] — interleaved in hop - # order so both shards advance core → frontier in parallel. - markers_paths = all_markers[shard_n::shard_m] - if not markers_paths: - log(f"error: shard {shard_n}/{shard_m} contains no bodies " - f"(full set = {len(all_markers)}).") - conn.close() - sys.exit(1) - - # Count distinct systems the shard will touch so the progress lines - # can report `systems X/Y done` alongside `bodies X/Y done`. - shard_systems = sorted({_body_id_from_path(p)[1] for p in markers_paths}) - total_shard_systems = len(shard_systems) + # Count distinct systems so the progress lines can report + # `systems X/Y done` alongside `bodies X/Y done`. + total_systems = len({_body_id_from_path(p)[1] for p in markers_paths}) seen_systems: set[str] = set() first_hop = hop_order.get(_body_id_from_path(markers_paths[0])[0], (99, ""))[0] last_hop = hop_order.get(_body_id_from_path(markers_paths[-1])[0], (99, ""))[0] - banner_shard = f"{shard_n}/{shard_m}" if shard_m > 1 else "single" log.raw("") log.raw(f" Gemma 2 Batch Naming Pipeline (#833)") log.raw(f" DB: {db_path}") log.raw(f" Mode: {'MOCK' if args.mock else 'REAL'}") - log.raw(f" Shard: {banner_shard}") log.raw(f" sr-voice: {MOCK_STDIO if args.mock else sr_voice_bin}") if not args.mock: log.raw(f" model: {model_path}") log.raw(f" seed: {args.seed} refresh: every {args.refresh} requests " f"stem-cap: {args.stem_cap}") - log.raw(f" {len(markers_paths)} markers.json files in this shard " - f"(full set: {len(all_markers)})") + log.raw(f" {len(markers_paths)} markers.json files to process") log.raw(f" hop {first_hop} → hop {last_hop}, core-first ordering") - log.raw(f" {total_shard_systems} distinct systems") + log.raw(f" {total_systems} distinct systems") log.raw(f" blocklist: {len(blocklist)} Earth-major entries") if log.log_path is not None: log.raw(f" log: {log.log_path}") log.raw("") + log.raw( + " Resume: re-run this command any time. Bodies whose markers.json " + "already has non-empty name fields will be skipped (preserved path)." + ) + log.raw("") corpus: dict[tuple[str, str], set[str]] = {} stem_counts: dict[str, int] = {} @@ -1362,7 +1319,7 @@ def main(): elapsed = time.time() - t0 body_progress = f"body {i+1}/{len(markers_paths)}" - sys_progress = f"sys {len(seen_systems)}/{total_shard_systems}" + sys_progress = f"sys {len(seen_systems)}/{total_systems}" hop = hop_order.get(body_id, (99, ""))[0] progress = f"{body_progress} {sys_progress} hop={hop}" @@ -1431,7 +1388,7 @@ def main(): eta = "--" log( f" >> CHECKPOINT bodies {i+1}/{len(markers_paths)} " - f"systems {len(seen_systems)}/{total_shard_systems} " + f"systems {len(seen_systems)}/{total_systems} " f"names {cum} {rate:.1f}/s eta {eta}" ) @@ -1445,10 +1402,9 @@ def main(): elapsed_total = time.time() - t_total log.raw("") log.raw(f" Done: {elapsed_total:.0f}s ({elapsed_total/60:.1f} min)") - log.raw(f" shard: {banner_shard}") log.raw(f" bodies processed: {len(markers_paths)}") log.raw(f" bodies touched: {bodies_touched}") - log.raw(f" systems seen: {len(seen_systems)}/{total_shard_systems}") + log.raw(f" systems seen: {len(seen_systems)}/{total_systems}") log.raw(f" cities named: {totals['cities']}") log.raw(f" rivers named: {totals['rivers']}") log.raw(f" oceans named: {totals['oceans']}")