diff --git a/src/helpers/question_curation.py b/src/helpers/question_curation.py index 82e3c11f..209a8308 100644 --- a/src/helpers/question_curation.py +++ b/src/helpers/question_curation.py @@ -13,7 +13,6 @@ manifold, metaculus, polymarket, - wikipedia, yfinance, ) @@ -69,11 +68,6 @@ "source_intro": fred.SOURCE_INTRO, "resolution_criteria": fred.RESOLUTION_CRITERIA, }, - "wikipedia": { - "name": "Wikipedia", - "source_intro": wikipedia.SOURCE_INTRO, - "resolution_criteria": wikipedia.RESOLUTION_CRITERIA, - }, "yfinance": { "name": "Yahoo Finance", "source_intro": yfinance.SOURCE_INTRO, diff --git a/src/helpers/wikipedia.py b/src/helpers/wikipedia.py index 234cf103..ffa9a21d 100644 --- a/src/helpers/wikipedia.py +++ b/src/helpers/wikipedia.py @@ -13,7 +13,6 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from sources._metadata import SOURCE_METADATA # noqa: E402 -from sources.wikipedia import _IDS_TO_NULLIFY as IDS_TO_NULLIFY # noqa: F401, E402 from sources.wikipedia import ( # noqa: F401, E402 _TRANSFORM_ID_MAPPING as transform_id_mapping, ) diff --git a/src/metadata/validate_questions/main.py b/src/metadata/validate_questions/main.py index 1da8b739..eec0972a 100644 --- a/src/metadata/validate_questions/main.py +++ b/src/metadata/validate_questions/main.py @@ -90,7 +90,7 @@ def validate_questions(dfq): @decorator.log_runtime def driver(_): """Pull in fetched data and update question metadata in question bank.""" - from helpers import question_curation, wikipedia + from helpers import question_curation local_filename = f"/tmp/{constants.META_DATA_FILENAME}" dfmeta = data_utils.download_and_read( @@ -120,11 +120,6 @@ def driver(_): "metaculus", ]: dfq["valid_question"] = True - if source == "wikipedia": - invalid_ids = set(wikipedia.transform_id_mapping.keys()) | { - entry["id"] for entry in wikipedia.IDS_TO_NULLIFY - } - dfq.loc[dfq["id"].isin(invalid_ids), "valid_question"] = False else: dfq = validate_questions(dfq) diff --git a/src/questions/wikipedia/update_questions/main.py b/src/questions/wikipedia/update_questions/main.py index 81ecb4c4..7068f970 100644 --- a/src/questions/wikipedia/update_questions/main.py +++ b/src/questions/wikipedia/update_questions/main.py @@ -48,7 +48,8 @@ def fill_missing_with_nan(df, dff): This could be for invalid reasons: a name change, e.g. Erigaisi Arjun -> Arjun Erigaisi Either way, fill these with nan. Invalid reasons will need to be caught by hand and - invalidated in `src/helpers/wikipedia.py` IDS_TO_NULLIFY. + invalidated by adding a `NullifiedQuestion` to the wikipedia `nullified_questions` list + in `src/sources/_metadata.py`. """ # fill in nan where the item has dropped out of the table all_dates = dff["date"].sort_values().unique() diff --git a/src/sources/wikipedia.py b/src/sources/wikipedia.py index 75cbd9d0..ee9c6fd5 100644 --- a/src/sources/wikipedia.py +++ b/src/sources/wikipedia.py @@ -15,7 +15,6 @@ from helpers import constants, dates from ._dataset import DatasetSource -from ._metadata import SOURCE_METADATA logger = logging.getLogger(__name__) @@ -216,8 +215,8 @@ def update(self, dfq, dff, **kwargs): # # If they were asked previously, they resolution values from the value variable are used. Hence # to be included in this list, the value must have been consistently used since the ID present - # in the `key` was first included in a question set. If not, then put the key in the - # `_IDS_TO_NULLIFY` list. + # in the `key` was first included in a question set. If not, then add a `NullifiedQuestion` for + # the key to the wikipedia `nullified_questions` list in `_metadata.py`. # # ******* # @@ -307,9 +306,3 @@ def update(self, dfq, dff, **kwargs): # "f9323386a651ce67fc0da31285bee22a4ec53b8a2ea5220431ecb4560fb44c77": "3f04d0cfccd38b26e86c0939516c483eb31edf6aaa3a1eaaabe38a48f7a0996a", } - - -_IDS_TO_NULLIFY = [ - {"id": nq.id, "nullify_start_date": nq.nullification_start_date} - for nq in SOURCE_METADATA["wikipedia"]["nullified_questions"] -] diff --git a/src/tests/test_question_curation_sources.py b/src/tests/test_question_curation_sources.py index 3e9cbb93..7399ad4f 100644 --- a/src/tests/test_question_curation_sources.py +++ b/src/tests/test_question_curation_sources.py @@ -1,7 +1,7 @@ -"""Sampling should exclude INFER; resolution/categorization should still include it.""" +"""Sampling excludes INFER and Wikipedia; the source registry still includes them for resolution.""" from helpers import question_curation -from sources import MARKET_SOURCE_NAMES +from sources import DATASET_SOURCE_NAMES, MARKET_SOURCE_NAMES def test_infer_not_sampled(): @@ -11,3 +11,12 @@ def test_infer_not_sampled(): def test_infer_still_a_market_source_for_resolution(): assert "infer" in MARKET_SOURCE_NAMES + + +def test_wikipedia_not_sampled(): + assert "wikipedia" not in question_curation.DATA_SOURCES + assert "wikipedia" not in question_curation.FREEZE_QUESTION_DATA_SOURCES + + +def test_wikipedia_still_a_dataset_source_for_resolution(): + assert "wikipedia" in DATASET_SOURCE_NAMES diff --git a/src/www.forecastbench.org/about/index.md b/src/www.forecastbench.org/about/index.md index 76b293bd..1b1a7870 100644 --- a/src/www.forecastbench.org/about/index.md +++ b/src/www.forecastbench.org/about/index.md @@ -15,7 +15,7 @@ footer_scripts:

We evaluate LLMs by regularly asking them to make probabilistic forecasts about future events, thereby creating a contamination-free benchmark.

We use two types of binary prediction questions: