{ "schema_version": 1, "protocol_id": "five-photo-model-choice-v1", "written_on": "2026-10-10", "status": "frozen_before_additional_photo_features", "executable": true, "question": "How sensitive is the five-item distance-graph assignment to the selected independently pretrained image and text encoders?", "scope": "A full three-by-two exploratory model-choice comparison using the same exhaustive graph objective. It is not Unpaired Rosetta, a learned cross-modal map, or an out-of-sample accuracy estimate.", "design_timing": "Specified after viewing the five photographs and after seeing the original DINO/GloVe outcome. New pair outcomes must remain unseen until a separate complete protocol and source/model/runtime lock are published. The earlier result remains part of the record.", "execution_gates": { "data_route": "same_five_existing_photographs", "user_choice": "The user explicitly requested comparing models on these five photographs first. No new collection is required for this extension.", "common_preparation": "Audit, acquire and hash the exact proposed model/source files; define and check inference on explicitly artificial fixtures without reading the private photographs or evaluating their labels.", "required_before_any_new_photo_inference": [ "Publish the exact binding to the original five-photo manifest and feature/result hashes.", "Replace every pending artifact/source/runtime/code hash slot with a verified receipt; publish a final lock covering all five encoders and the common solver.", "Complete and retain the fixed synthetic implementation gates for every new encoder. Tests verify computation, never choose a representation by correspondence accuracy.", "Freeze all representations, preprocessing, numerical tolerances, pair order, reporting and failure rules before looking at any new pair outcome.", "Create a separately named executable protocol. This draft itself never authorizes an experiment or substitutes for a final lock." ], "current_completion": "All model files acquired and byte-locked; synthetic runtime and numerical gates passed. Publish this protocol and its release lock before any new study-item feature extraction." }, "relationship_to_primary": { "protocol_id": "five-photo-geometry-v1", "source_commit": "feead1da4d77bb0dc674fcf0f5f38979fac9a90c", "protocol_sha256": "fcc1c4252ae268e7dcf645419eaea07e1196c09e939a7066e55cf084afddf603", "extractor_sha256": "37665c8dbb0d2479bfc162854ad0508d6e1b89a6dc8ee51f75bb4d095c14a823", "solver_sha256": "6819921cecd461dc841b1ea89dddb18fab53fc0c36c07b631d7f755346ce5dfc", "feature_helper_sha256": "3e8445f22a8f255dcf47af89ac0b8c45461ba98046549702abc7b09b6b59a624", "same_five_policy": "Reuse the verified existing DINO and GloVe feature rows and their published DINO/GloVe result unchanged. Check the original feature and input hashes and row identities. Do not rerun, recrop, recenter or reinterpret that primary analysis to improve its outcome.", "other_work": "The paused EPIC experiment and the earlier 48-photograph competence protocol remain unchanged and are not resumed by this draft." }, "data": { "same_five_existing_photographs": { "items": 5, "candidate_ids": [ "candidate-001", "candidate-002", "candidate-003", "candidate-004", "candidate-005" ], "human_labels_verbatim": [ "Slide", "Trashcan", "Tree", "Basketball", "Water fountain" ], "selection": "Every photograph and exact human label from the original private manifest, including ambiguous object/activity labels. No new exclusion, crop, label rewrite, prompt expansion, synonym substitution or additional view.", "attachment_byte_identity": "Require equality with the original private image hashes and manifest before extracting any alternative model. These are supplied attachment bytes; they are not an independently authenticated camera-original archive.", "private_manifest_sha256": "252eab59d902fcb3a772a73782d46156d6f53c9bf67bc3f79010b52e95287ece", "original_feature_npz_sha256": "d9924e021e2b4233493930454ca7d32be2cde55617d1ed8aaca83bf11b528515", "original_feature_metadata_sha256": "52329157770a80d6f751e7af3bae90da985ddd528940524f334ee85146b554a0", "original_selected_result_sha256": "a1e438fbe79ce42bae93e92e6139e825e7ac368c56abcfe7e5221ad3e9f19b11", "original_evaluation_sha256": "6561389d21da6a9450205cf9fc158b291ea17af8ca1253c2ec129b13606cfe01", "binding_status": "Bound to the original private artifact hashes. The first result remains unchanged." }, "use_of_all_items": "There is no fitting/test split in this graph comparison. All five image embeddings and the complete five-label candidate set determine each assignment. The comparison is transductive and closed-set.", "privacy": "Photographs, identifying capture metadata, private manifests and raw features remain private. Public output may contain generic labels, opaque identifiers, distance matrices, assignments, numerical summaries and hashes." }, "chronology": { "original_pair": "The original DINO/GloVe files were hash-locked before the collector reports taking the existing photographs. The capture chronology is a human attestation, not an independently authenticated event timeline.", "added_models_on_existing_photos": "MoCo, MAE and ELMo were shortlisted by provenance before the primary result, but their full weight bytes were not locked before these photographs. Newly downloaded alternative files cannot inherit the original pair's pre-capture byte-lock claim. Interpret their chronology through separately documented release/training evidence, with that weaker basis disclosed.", "added_models_on_future_photos": "A published complete panel lock can precede future human capture. It supports exclusion of those later photographed events from the already fixed encoders, conditional on honest capture records and use of the locked bytes.", "not_excluded": [ "shared familiar concepts", "ordinary label words and phrases", "similar scenes or physical object types", "human ImageNet curation", "researcher choice of models and tasks", "the observed original result influencing this extension's design" ] }, "shared_image_preprocessing": { "helper_sha256": "3e8445f22a8f255dcf47af89ac0b8c45461ba98046549702abc7b09b6b59a624", "recipe": "Apply EXIF orientation; convert RGB; resize short side to 256 using Pillow bicubic with integer-floor long side; center crop 224 with round((dimension-224)/2); scale to [0,1]; normalize with ImageNet mean [0.485,0.456,0.406] and std [0.229,0.224,0.225].", "control": "Feed exactly the same preprocessed tensor bytes to every image encoder. No model-specific crop, augmentation, test-time ensemble or adaptive view selection.", "arithmetic": "FP32 evaluation, no gradients or model training, batch size one, deterministic algorithms, TF32 disabled; record the exact runtime and device.", "moco_comparison_note": "The common frozen bicubic resize differs from the historical MoCo linear-evaluation script's bilinear resize. No crop or interpolation method is selected by results." }, "vision_models": [ { "id": "dino_v1_vit_b16", "name": "Original DINOv1 ViT-B/16", "documented_pretraining": "ImageNet-1K; self-distillation without class-label targets", "checkpoint_sha256": "bf34ad0f424b9029b593e8dc3ed553bf26e88bcba0d32bf3e62a6209cb64c85e", "source_commit": "7c446df5b9f45747937fb0d72314eb9f7b66930a", "source_sha256": { "vision_transformer.py": "b1f998d5f49ab43666b9fc6d007c5f6540c3ead2e8645e144f74045f63ff44d7", "utils.py": "962a97e1acda1c986dfd921275325e4083f9016dddadcf018f1ccf03fa600eab" }, "dimensions": 768, "representation": "Final-block layer-normalized CLS, followed by row L2 normalization, exactly as the original five-photo helper.", "same_five_status": "Reuse existing verified feature bytes; no new representation choice." }, { "id": "moco_v2_resnet50_800ep", "name": "Original MoCo-v2 ImageNet ResNet-50, 800 epochs", "documented_pretraining": "ImageNet-1K; contrastive training of augmented image views", "checkpoint_url": "https://dl.fbaipublicfiles.com/moco/moco_checkpoints/moco_v2_800ep/moco_v2_800ep_pretrain.pth.tar", "checkpoint_sha256": "5e4a3fd7178a95837c0ad71c265ed5f477d271e753ba505b6b6a5e86829eeb7e", "checkpoint_receipt": { "file": "vision-artifacts.json", "sha256": "84d0a32f8678e4d3e5fbce173b54262b0c74ccb19a637e22de35c00210cf9bfb" }, "source_commit": "78b69cafae80bc74cd1a89ac3fb365dc20d157d3", "known_source_sha256": { "main_lincls.py": "c3fae8e33de5fac9a589ef4f5464184604f1d6e55081c4406576f54c98558b47", "moco/builder.py": "349f418a8502e3a4fe5428535b375e5517f4df8591614d17168f91b4900ff1b2" }, "architecture_implementation_lock": { "torchvision_version": "0.22.1+cu128", "resnet_source_sha256": "7499a5059ad7b1a1f8f7543c04b4a13795140fad837c38530b48ff5d9610bf6e" }, "dimensions": 2048, "representation": "Query encoder's ResNet-50 backbone after global average pooling and before the projection/classification head; row L2 normalization. Use frozen running batch-normalization statistics in evaluation mode.", "load_rule": "Load all query-backbone parameters and buffers with exact expected key coverage. Exclude the momentum key encoder, queue, contrastive projection head and downstream classifier. Enumerate any intentionally excluded keys; no silent missing backbone weights.", "excluded_alternatives": [ "MoCo v1", "200-epoch MoCo-v2 checkpoint", "Instagram-trained checkpoint", "downstream fine-tuned or classifier feature weights" ] }, { "id": "mae_vit_b16_pretrain_cls", "name": "Original MAE ViT-B/16 pretrained encoder, CLS readout", "documented_pretraining": "ImageNet-1K; masked pixel reconstruction", "checkpoint_url": "https://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_base.pth", "checkpoint_sha256": "aec5f0b68e5f3193a00b07bc65a37440db549c15b36b8bea242606cc40c4bc5d", "checkpoint_receipt": { "file": "vision-artifacts.json", "sha256": "84d0a32f8678e4d3e5fbce173b54262b0c74ccb19a637e22de35c00210cf9bfb" }, "source_commit": "efb2a8062c206524e35e47d04501ed4f544c0ae8", "known_source_sha256": { "models_mae.py": "973c01f31fdf15febb5976baecd639b83a5cc011161af852e54ca68c6c2bafb4" }, "models_vit_source_sha256": "6770ed2226d46ab8ba2b1ebc6662cc353dabf759679569acd004f6b3ac868154", "main_linprobe_source_sha256": "bc4eab531eb32538cb8cc572c2a7431ddb587ff8be848e8b4b2e4bd73788b7a2", "timm_or_equivalent_implementation_lock": { "vendor_lock_file": "vision-vendor-lock.json", "vendor_lock_sha256": "dea029b7f933a2acb9b8f6f0930bed63813c6bd0499e2ccde5ac14732ceea12b", "wheel_sha256": "c1598ef61c246e38835a6b834811dcb8d62ae56b644cd54dc3ef9565c8387875", "compatibility_change": "temporary torch._six.container_abcs alias to collections.abc during timm import only" }, "dimensions": 768, "representation": "All 196 original-order patch tokens plus CLS, no masking or patch shuffle; original positional embeddings and all encoder blocks; final encoder layer norm; CLS token only; row L2 normalization. No learned head or head batch-normalization transform.", "readout_rationale": "The official linear-probe entry point sets global_pool=False, and its models_vit.forward_features applies the encoder norm then returns the CLS token. This is a source-based fixed choice, not a choice between CLS and pooled patches made from these photographs' results.", "load_rule": "Use the original pretrained encoder, without supervised fine-tuning. Check every encoder parameter and buffer against the checkpoint; explicitly account for unused decoder/mask tokens and excluded classifier keys. Preserve original positional embeddings at 224 pixels; no interpolation is needed.", "masking_trap": "The original models_mae.forward_encoder calls random_masking even when mask_ratio=0. Do not use that path as deterministic unmasked extraction. Use the official unmasked models_vit CLS path or an explicitly verified equivalent in original patch order.", "source_evidence": [ "https://raw.githubusercontent.com/facebookresearch/mae/efb2a8062c206524e35e47d04501ed4f544c0ae8/main_linprobe.py", "https://raw.githubusercontent.com/facebookresearch/mae/efb2a8062c206524e35e47d04501ed4f544c0ae8/models_vit.py", "https://raw.githubusercontent.com/facebookresearch/mae/efb2a8062c206524e35e47d04501ed4f544c0ae8/models_mae.py" ] } ], "shared_text_input": { "labels": "Use the five exact human label strings without prompts, expansions or semantic rewriting.", "tokenizer": { "unicode_normalization": "NFKC", "case": "lower", "regex": "[a-z]+(?:'[a-z]+)?|[0-9]+" }, "tokens": "Retain repetitions and stopwords. The input token sequence is the same for both text models; GloVe sorts occurrences only for averaging arithmetic, whereas ELMo retains their original sequence order.", "empty_rule": "Stop on an empty token sequence or invalid feature; do not drop or relabel an item." }, "text_models": [ { "id": "glove_original_6b_300d", "name": "Original GloVe 6B, 300 dimensions", "documented_pretraining": "Wikipedia 2014 and Gigaword 5", "archive_sha256": "6471382cdd837544bf3ac72497a38715e845897d265b2b424b4761832009c837", "member": "glove.6B.300d.txt", "member_sha256": "a12599d41e3589c7160be27fffe5b0080eccd0f0c75f46666c59f90188093c40", "dimensions": 300, "representation": "Unweighted float32 mean of sorted known token occurrences, retaining repeats and stopwords, followed by L2 normalization. Omit and report unknown tokens; fail when no known tokens remain or the vector is zero/nonfinite.", "same_five_status": "Reuse existing verified feature bytes, token report and original tokenizer/pooling." }, { "id": "elmo_original_1b_top_context", "name": "Original ELMo 1 Billion Word Benchmark model, top contextual layer", "documented_pretraining": "1 Billion Word Benchmark derived from WMT 2011 News Crawl", "artifact_name": "2x4096_512_2048cnn_2xhighway", "weights_url": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5", "weights_sha256": "a6b92b9ccbb13fe489faacd2221c795d4bebe84b75f8aded9d85ea464fa5c994", "weights_receipt": { "file": "elmo-artifacts.json", "sha256": "4f90865834f953a7741762ac7cd8f68fea225918e7755ecfac455fb2e5e6bbfc" }, "options_url": "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json", "options_sha256": "86cb4c1a7e2f25e3867d06810e33ae27fbbd511ac77ca377be5fb7405a7085b1", "reference_source_commit": "7cffee2b0986be51f5e2a747244836e1047657f4", "known_source_sha256": { "bilm/model.py": "b6bb9158c2e3c3897e62748784165c5126eb42dc36293a540689d5a8b2cea5e0", "bilm/elmo.py": "80dfb939bc515c629b278d36260d7108e7421c2ea79fb9b4f905f36949b63a58" }, "character_mapping_source_lock": { "allennlp_version": "2.10.1", "elmo_indexer_sha256": "28e3ef2ff9a9e564c07e574ef63a28f25fe2d9757518971c604ae2b8d7440ae7" }, "executable_implementation_lock": { "wrapper_sha256": "0fff41c3740a7ece2f89a4b9ba4b50d2d370606357f2ff57cce310a12e3d09ce", "backend_source_sha256": { "modules/elmo.py": "a02b3ce51773fa125ae889affc21a65201bd8451bee997b3e956d9cb9f0d5c90", "modules/elmo_lstm.py": "aec79417f2d60d722847bb861475edafee07bc163a1aeee2f70764d5d5290056", "modules/encoder_base.py": "373b14173415ba1180c9f1544ea7762e3b263c698b30bd486d8cb175c529b9d0", "modules/lstm_cell_with_projection.py": "66f84517e5fe4ba3b78a955580bc4e206fdb470b2c6d0a05039af5d9c89f03cf", "modules/highway.py": "47d270ab4f478426121c476922b1948f995228f171955b270031a5fd3c90daf7", "modules/scalar_mix.py": "9efb83375d4ef2d9f56ba464e8db2a13d3e529fd33115096bdf14fd345dc96eb", "nn/util.py": "ace9269f2b80170358abb9006ef4c9b3bc7886828c49c24b9e33b1e41397345e", "data/token_indexers/elmo_indexer.py": "28e3ef2ff9a9e564c07e574ef63a28f25fe2d9757518971c604ae2b8d7440ae7" }, "runtime": { "allennlp": "2.10.1", "torch": "1.12.1+cpu", "numpy": "1.23.5", "h5py": "3.8.0", "python": "3.10.19", "system": "Linux", "machine": "x86_64", "device": "cpu", "threads": 4, "interop_threads": 1, "deterministic_algorithms": true }, "environment_receipt": { "file": "elmo-environment.json", "sha256": "d74fa50cd856a8eff7dce6f336988b3a2c18edc9fc4cff833b2295d7b4834d6d" } }, "dimensions": 1024, "representation": "Character-input original bidirectional language model; top (second) contextual LSTM layer only, concatenated 512-dimensional directional outputs. Mean in original token order over real input token positions only, excluding padding and boundary markers; FP32 arithmetic; row L2 normalization.", "state_rule": "Start each independent label with zero recurrent hidden and cell states in both directions. No state persists across items, batches, models or repetitions. Disable dropout and training updates. Keep the label's original token order.", "boundary_rule": "Use the pinned reference model's normal sentence boundary tokens for context, but never include boundary or padding positions in the pooled representation.", "truncation_rule": "Record character mapping and any token truncation under the pinned implementation. Stop rather than silently truncate or replace a supplied label token. Character input does not eliminate the need to report mapping limitations.", "excluded_alternatives": [ "Original ELMo 5.5B", "task-fine-tuned ELMo or SNLI models", "learned or task-borrowed scalar layer mixture", "top-performing layer selection", "state carried over from earlier labels" ] } ], "full_factorial": { "vision_order": [ "dino_v1_vit_b16", "moco_v2_resnet50_800ep", "mae_vit_b16_pretrain_cls" ], "text_order": [ "glove_original_6b_300d", "elmo_original_1b_top_context" ], "required_cells": [ { "vision": "dino_v1_vit_b16", "text": "glove_original_6b_300d", "question": "Preserved original reference on the same five inputs; no re-selection." }, { "vision": "dino_v1_vit_b16", "text": "elmo_original_1b_top_context", "question": "Does replacing the language encoder change the selected correspondence while the image representation is fixed?" }, { "vision": "moco_v2_resnet50_800ep", "text": "glove_original_6b_300d", "question": "Does replacing DINO with MoCo change correspondence while GloVe is fixed?" }, { "vision": "moco_v2_resnet50_800ep", "text": "elmo_original_1b_top_context", "question": "Does the language-encoder contrast also appear with the MoCo image representation?" }, { "vision": "mae_vit_b16_pretrain_cls", "text": "glove_original_6b_300d", "question": "Does replacing DINO with MAE change correspondence within the ViT-B/16 backbone family while GloVe is fixed?" }, { "vision": "mae_vit_b16_pretrain_cls", "text": "elmo_original_1b_top_context", "question": "Does the language-encoder contrast also appear with the MAE image representation?" } ], "hypothesis_status": "All six are descriptive sensitivity questions. No directional improvement prediction, significance test, winner-selection rule or claim of six independent datasets.", "contrasts": "Show the language replacement within each image model and image replacements within each language model. Report changes in assignments, correctness, human-assignment rank and tie structure. Do not choose a subset of cells after results.", "causal_limit": "Model replacements jointly change multiple properties. DINO versus MoCo changes architecture and objective; DINO versus MAE also changes positional embeddings and training recipe despite sharing a backbone family; GloVe versus ELMo changes corpus, objective, dimensionality and representation. These are not isolated causal effects of architecture, loss function or word order." }, "matching_analysis": { "feature_arithmetic": "Keep frozen model outputs as FP32; convert each row to float64 and L2 normalize again only for distance arithmetic. No centering, whitening, PCA, learned metric, layer sweep or additional normalization.", "distance": "For each modality compute d(i,j)=1-dot(unit_i,unit_j); symmetrize and set the diagonal to zero. Preserve unrounded float64 values.", "image_shuffle": { "implementation": "Python random.Random(seed).shuffle of manifest indices 0 through 4", "seed": 20261010 }, "text_shuffle": { "implementation": "Python random.Random(seed).shuffle of manifest indices 0 through 4", "seed": 20261011 }, "shared_order": "Use the same private manifest order and the same two shuffled index lists for all six cells. Opaque IDs I00..I04 and T00..T04 must refer to the same underlying items across models.", "solver_input": "Only the two 5-by-5 distance matrices, opaque IDs and protocol/feature provenance. No labels, photographs or correct pairings.", "permutations_per_cell": 120, "objective": "Mean, over the ten unordered image pairs i