| """debug_downloaded_data.py - Inspect the downloaded conversation format"""
|
|
|
| import json
|
|
|
| def inspect_downloaded_data():
|
| """Inspect the first few records to understand the format"""
|
|
|
| data_path = "data/conversation_raw/OpenAssistant_oasst1_raw.jsonl"
|
|
|
| print("🔍 Inspecting downloaded OpenAssistant data...")
|
| print("="*50)
|
|
|
| try:
|
| with open(data_path, 'r', encoding='utf-8') as f:
|
| for i in range(5):
|
| line = f.readline().strip()
|
| if line:
|
| record = json.loads(line)
|
| print(f"\nRecord {i+1}:")
|
| print(f"Top-level keys: {list(record.keys())}")
|
|
|
|
|
| for key, value in record.items():
|
| if isinstance(value, str) and len(value) > 100:
|
| value = value[:100] + "..."
|
| elif isinstance(value, dict):
|
| value = f"Dict with keys: {list(value.keys())}"
|
| elif isinstance(value, list):
|
| value = f"List with {len(value)} items"
|
|
|
| print(f" {key}: {value}")
|
|
|
|
|
| for key in ['prompt', 'conversation', 'messages']:
|
| if key in record and isinstance(record[key], (dict, list)):
|
| print(f"\n Exploring {key}:")
|
| nested = record[key]
|
| if isinstance(nested, dict):
|
| print(f" Keys: {list(nested.keys())}")
|
| for nkey, nvalue in list(nested.items())[:3]:
|
| if isinstance(nvalue, str) and len(nvalue) > 50:
|
| nvalue = nvalue[:50] + "..."
|
| print(f" {nkey}: {nvalue}")
|
| elif isinstance(nested, list) and nested:
|
| print(f" First item type: {type(nested[0])}")
|
| if isinstance(nested, dict):
|
| print(f" First item keys: {list(nested.keys())}")
|
|
|
| except Exception as e:
|
| print(f"Error reading file: {e}")
|
|
|
| if __name__ == "__main__":
|
| inspect_downloaded_data()
|
|
|
