Figure 12 Stage 1: tag-based filtering used to build the unsafe test set.
Python code for tag-based filtering
def gopher_rules_pass(sample) -> bool:
""" function returns True if the sample complies with Gopher rules """
signals = json.loads(sample["quality_signals"])
# rule 1: number of words between 50 and 10'000
if "rps_doc_word_count" not in signals.keys():
return False
word_count = signals["rps_doc_word_count"][0][2]
if word_count < 50 or word_count > 100_000:
return False
# rule 2: mean word length between 3 and 10
mean_word_length = signals["rps_doc_mean_word_length"][0][2]
if mean_word_length < 3 or mean_word_length > 10:
return False
# rule 2: symbol to word ratio below 0.1
symbol_word_ratio = signals["rps_doc_symbol_to_word_ratio"][0][2]
if symbol_word_ratio > 0.1:
return False
# rule 3: 90% of lines need to start without a bullet point
n_lines = signals["ccnet_nlines"][0][2]
n_lines_bulletpoint_start = sum(map(lambda ln: ln[2], signals["rps_lines_start_with_bulletpoint"]))
if n_lines_bulletpoint_start / n_lines > 0.9:
return False
# rule 4: the ratio between characters in the most frequent 2-gram and the total number
# of characters must be below 0.2
top_2_gram_frac = signals["rps_doc_frac_chars_top_2gram"][0][2]
if top_2_gram_frac > 0.2:
return False
# rule 5: curse words or blocklist content should be present
curse_words = signals["rps_doc_ldnoobw_words"][0][2]
# https://data.together.xyz/redpajama-data-v2/v1.0.0/artifacts/ut1_domain_categories.json
blocklist_content = signals["rps_doc_ut1_blacklist"][0][2] is None
if curse_words < 5.0 or blocklist_content:
return False
return True
%\end{minted}
26