help with a better solution for generating delta log from table stats

[djouallah]

I am using this hack to generate delta metadata file, I am only interested in the latest state of the table, and assuming there is no deletion file, I am sharing it just to get better ideas, maybe getting the file schema and stats from the duckdb metadata too

def map_type(pa_field):
    t = pa_field.type
    if t.id == pa.int32().id:
        return "integer"
    elif t.id == pa.int16().id:
        return "short"
    elif t.id == pa.int64().id:
        return "long"
    elif t.id == pa.float32().id:
        return "float"
    elif t.id == pa.float64().id:
        return "double"
    elif t.id == pa.string().id:
        return "string"
    elif t.id == pa.bool_().id:
        return "boolean"
    elif t.id == pa.date32().id:
        return "date"
    elif t.id == pa.timestamp('ns').id:
        return "timestamp"
    else:
        raise NotImplementedError(f"Unsupported Arrow type: {t}")


def pyarrow_schema_to_delta_json(pa_schema):
    return {
        "type": "struct",
        "fields": [{
            "name": f.name,
            "type": map_type(f),
            "nullable": f.nullable,
            "metadata": {}
        } for f in pa_schema]
    }


def generate_latest_delta_log(schema,tbl):
    table_path = f"/lakehouse/default/Tables/{schema}/{tbl}"
    delta_log = os.path.join(table_path, "_delta_log")
    parquet_files = con.sql(f" select data_file from ducklake_list_files('db', '{tbl}');").df()['data_file'].to_list()
    
   
    log_file = os.path.join(delta_log, "00000000000000000000.json")

    # Write new log file
    protocol = {"protocol": {"minReaderVersion": 1, "minWriterVersion": 2}}
    schema = pq.read_schema(parquet_files[0])
    metadata = {
        "metaData": {
            "id": str(uuid.uuid4()),
            "format": {"provider": "parquet", "options": {}},
            "schemaString": json.dumps(pyarrow_schema_to_delta_json(schema)),
            "partitionColumns": [],
            "createdTime": int(time.time() * 1000),
            "configuration": {}
        }
    }

    adds = []
    for f in parquet_files:
        adds.append({
            "add": {
                "path": os.path.relpath(f, table_path).replace("\\", "/"),
                "partitionValues": {},
                "size": os.path.getsize(f),
                "modificationTime": int(os.path.getmtime(f) * 1000),
                "dataChange": True,
                "stats": None
            }
        })

    # Ensure _delta_log directory exists
    os.makedirs(delta_log, exist_ok=True)

    # Write new log file
    with open(log_file, "w") as f:
        f.write(json.dumps(protocol) + "\n")
        f.write(json.dumps(metadata) + "\n")
        for a in adds:
            f.write(json.dumps(a) + "\n")
    print("Delta log updated successfully.")

[djouallah]

closing as i found a better option

[arouel]

Can you share your better solution here?

[djouallah]

same thing, slightly less horrible

#164