fix: stop sampling wikipedia questions into new question sets

src/helpers/question_curation.py

@@ -13,7 +13,6 @@
     manifold,
     metaculus,
     polymarket,
-    wikipedia,
     yfinance,
 )
 
@@ -69,11 +68,6 @@
         "source_intro": fred.SOURCE_INTRO,
         "resolution_criteria": fred.RESOLUTION_CRITERIA,
     },
-    "wikipedia": {
-        "name": "Wikipedia",
-        "source_intro": wikipedia.SOURCE_INTRO,
-        "resolution_criteria": wikipedia.RESOLUTION_CRITERIA,
-    },
     "yfinance": {
         "name": "Yahoo Finance",
         "source_intro": yfinance.SOURCE_INTRO,

src/helpers/wikipedia.py

@@ -13,7 +13,6 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
 from sources._metadata import SOURCE_METADATA  # noqa: E402
-from sources.wikipedia import _IDS_TO_NULLIFY as IDS_TO_NULLIFY  # noqa: F401, E402
 from sources.wikipedia import (  # noqa: F401, E402
     _TRANSFORM_ID_MAPPING as transform_id_mapping,
 )

src/metadata/validate_questions/main.py

@@ -90,7 +90,7 @@ def validate_questions(dfq):
 @decorator.log_runtime
 def driver(_):
     """Pull in fetched data and update question metadata in question bank."""
-    from helpers import question_curation, wikipedia
+    from helpers import question_curation
 
     local_filename = f"/tmp/{constants.META_DATA_FILENAME}"
     dfmeta = data_utils.download_and_read(
@@ -120,11 +120,6 @@ def driver(_):
             "metaculus",
         ]:
             dfq["valid_question"] = True
-            if source == "wikipedia":
-                invalid_ids = set(wikipedia.transform_id_mapping.keys()) | {
-                    entry["id"] for entry in wikipedia.IDS_TO_NULLIFY
-                }
-                dfq.loc[dfq["id"].isin(invalid_ids), "valid_question"] = False
         else:
             dfq = validate_questions(dfq)
 

src/questions/wikipedia/update_questions/main.py

@@ -48,7 +48,8 @@ def fill_missing_with_nan(df, dff):
         This could be for invalid reasons: a name change, e.g. Erigaisi Arjun -> Arjun Erigaisi
 
         Either way, fill these with nan. Invalid reasons will need to be caught by hand and
-        invalidated in `src/helpers/wikipedia.py` IDS_TO_NULLIFY.
+        invalidated by adding a `NullifiedQuestion` to the wikipedia `nullified_questions` list
+        in `src/sources/_metadata.py`.
         """
         # fill in nan where the item has dropped out of the table
         all_dates = dff["date"].sort_values().unique()

src/sources/wikipedia.py

@@ -15,7 +15,6 @@
 from helpers import constants, dates
 
 from ._dataset import DatasetSource
-from ._metadata import SOURCE_METADATA
 
 logger = logging.getLogger(__name__)
 
@@ -216,8 +215,8 @@ def update(self, dfq, dff, **kwargs):
     #
     # If they were asked previously, they resolution values from the value variable are used. Hence
     # to be included in this list, the value must have been consistently used since the ID present
-    # in the `key` was first included in a question set. If not, then put the key in the
-    # `_IDS_TO_NULLIFY` list.
+    # in the `key` was first included in a question set. If not, then add a `NullifiedQuestion` for
+    # the key to the wikipedia `nullified_questions` list in `_metadata.py`.
     #
     # *******
     #
@@ -307,9 +306,3 @@ def update(self, dfq, dff, **kwargs):
     #
     "f9323386a651ce67fc0da31285bee22a4ec53b8a2ea5220431ecb4560fb44c77": "3f04d0cfccd38b26e86c0939516c483eb31edf6aaa3a1eaaabe38a48f7a0996a",
 }
-
-
-_IDS_TO_NULLIFY = [
-    {"id": nq.id, "nullify_start_date": nq.nullification_start_date}
-    for nq in SOURCE_METADATA["wikipedia"]["nullified_questions"]
-]

src/tests/test_question_curation_sources.py

@@ -1,7 +1,7 @@
-"""Sampling should exclude INFER; resolution/categorization should still include it."""
+"""Sampling excludes INFER and Wikipedia; the source registry still includes them for resolution."""
 
 from helpers import question_curation
-from sources import MARKET_SOURCE_NAMES
+from sources import DATASET_SOURCE_NAMES, MARKET_SOURCE_NAMES
 
 
 def test_infer_not_sampled():
@@ -11,3 +11,12 @@ def test_infer_not_sampled():
 
 def test_infer_still_a_market_source_for_resolution():
     assert "infer" in MARKET_SOURCE_NAMES
+
+
+def test_wikipedia_not_sampled():
+    assert "wikipedia" not in question_curation.DATA_SOURCES
+    assert "wikipedia" not in question_curation.FREEZE_QUESTION_DATA_SOURCES
+
+
+def test_wikipedia_still_a_dataset_source_for_resolution():
+    assert "wikipedia" in DATASET_SOURCE_NAMES

src/www.forecastbench.org/about/index.md

@@ -15,7 +15,7 @@ footer_scripts:
       <p>We evaluate LLMs by regularly asking them to make probabilistic forecasts about future events, thereby creating a contamination-free benchmark.</p>
       <p>We use two types of binary prediction questions:
       <ul>
-      <li><strong>Dataset questions</strong> are automatically generated from real-world time series (<a href="https://acleddata.com/" class="no-wrap">ACLED <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://db.nomics.world/" class="no-wrap">DBnomics <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://fred.stlouisfed.org/" class="no-wrap">FRED <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://finance.yahoo.com/" class="no-wrap">Yahoo! Finance <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, and <a href="https://www.wikipedia.org/" class="no-wrap">Wikipedia <i class="fa-solid fa-arrow-up-right-from-square"></i></a>) using pre-specified templates. Each dataset question generates multiple forecasts at different time horizons, since we ask the same question with 8 different resolution dates, ranging from 7 days to 10 years out.</li>
+      <li><strong>Dataset questions</strong> are automatically generated from real-world time series (<a href="https://acleddata.com/" class="no-wrap">ACLED <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://db.nomics.world/" class="no-wrap">DBnomics <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://fred.stlouisfed.org/" class="no-wrap">FRED <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, and <a href="https://finance.yahoo.com/" class="no-wrap">Yahoo! Finance <i class="fa-solid fa-arrow-up-right-from-square"></i></a>) using pre-specified templates. Each dataset question generates multiple forecasts at different time horizons, since we ask the same question with 8 different resolution dates, ranging from 7 days to 10 years out.</li>
       <li><strong>Market questions</strong> are drawn from leading prediction platforms: <a href="https://kalshi.com/" class="no-wrap">Kalshi <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://manifold.markets/" class="no-wrap">Manifold <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, <a href="https://www.metaculus.com/" class="no-wrap">Metaculus <i class="fa-solid fa-arrow-up-right-from-square"></i></a>, and <a href="https://polymarket.com/" class="no-wrap">Polymarket <i class="fa-solid fa-arrow-up-right-from-square"></i></a>.</li>
       </ul>
       </p>