Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1435,6 +1435,13 @@ def primitive(self, primitive: pa.DataType) -> PrimitiveType:
elif pa.types.is_null(primitive):
# PyArrow null type (pa.null()) is converted to Iceberg UnknownType
# UnknownType can be promoted to any primitive type in V3+ tables per the Iceberg spec
if self._format_version < 3:
field_path = ".".join(self._field_names) if self._field_names else "<root>"
raise ValueError(
"Null type (pa.null()) is not supported in Iceberg format version "
f"{self._format_version}. Field: {field_path}. "
"Use a concrete type (string, int, boolean, etc.) or set format-version=3."
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: wdyt about future proofing with a message like: Requires v3+ or use a concrete type.

)
return UnknownType()
elif isinstance(primitive, pa.UuidType):
return UUIDType()
Expand Down
10 changes: 10 additions & 0 deletions tests/catalog/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,16 @@ def test_convert_schema_if_needed(
assert expected == catalog._convert_schema_if_needed(schema)


def test_convert_schema_if_needed_rejects_null_type(catalog: InMemoryCatalog) -> None:
schema = pa.schema([pa.field("n1", pa.null())])
with pytest.raises(ValueError) as exc_info:
catalog._convert_schema_if_needed(schema)
message = str(exc_info.value)
assert "Null type" in message
assert "n1" in message
assert "format-version=3" in message
Comment on lines +215 to +217
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is the full error message

'Null type (pa.null()) is not supported in Iceberg format version 2. Field: n1. Use a concrete type (string, int, boolean, etc.) or set format-version=3.'



def test_create_table_pyarrow_schema(catalog: InMemoryCatalog, pyarrow_schema_simple_without_ids: pa.Schema) -> None:
catalog.create_namespace(TEST_TABLE_NAMESPACE)
table = catalog.create_table(
Expand Down