https://github.com/emdgroup/foundry-dev-tools
uv pip install foundry-dev-tools-transforms 'foundry-dev-tools[full]' pandas polars ipython pyarrow s3fs xarray zarr
import os
import toml
from pathlib import Path
try:
jwt_token = os.environ["FOUNDRY_TOKEN"]
except KeyError:
jwt_token = os.environ["PALANTIR_API_TOKEN"]
try:
foundry_external_host = os.environ["FOUNDRY_EXTERNAL_HOST"]
domain = foundry_external_host.split("//")[-1] if foundry_external_host else None
except:
domain = os.environ["PALANTIR_FOUNDRY_HOSTNAME"]
config = {"credentials": {"domain": domain, "jwt": jwt_token}}
file_path = Path.home() / ".foundry-dev-tools" / "config.toml"
file_path.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, "w") as f:
toml.dump(config, f)
Get a list of datasets
from foundry_dev_tools import FoundryContext
ctx = FoundryContext()
children = list(ctx.compass.get_child_objects_of_folder("ri.compass.main.folder.ID"))
data = {f["name"]: {"rid": f["rid"]} for f in children}
List ontologies
from foundry_dev_tools import FoundryContext
from foundry_dev_tools.clients.public_ontologies_client import OntologiesClient
ctx = FoundryContext()
ontologies_client = OntologiesClient(ctx)
ontologies_client.list()
response = ontologies_client.api_list_ontologies()
from foundry_dev_tools.resources.resource import Resource
Resource.from_rid(ctx, "ri.ontology.main.object-type.ID")
Read data
from foundry_dev_tools import FoundryContext
ctx = FoundryContext()
s3_resource = ctx.s3.get_boto3_resource()
ds_rid = "ri.foundry.main.dataset.ID"
bucket = s3_resource.Bucket(ds_rid)
for obj in bucket.objects.all():
print(obj.key)
s3_file = "s3://ri.foundry.main.dataset.ID/ID-my_output.parquet"
import pandas as pd
storage_options = ctx.s3.get_s3fs_storage_options()
df = pd.read_parquet(s3_file, storage_options=storage_options)
import polars as pl
storage_options = ctx.s3.get_polars_storage_options()
df = pl.read_parquet(s3_file, storage_options=storage_options)
import xarray as xr
storage_options = ctx.s3.get_s3fs_storage_options()
s3_file = "s3://ri.foundry.main.dataset.ID/ray.bell/air_temperature.zarr"
ds = xr.open_zarr(s3_file, storage_options=storage_options)
Upload a folder
from foundry_dev_tools import FoundryContext
from pathlib import Path
ctx = FoundryContext()
dataset = ctx.get_dataset_by_path("ri.compass.main.folder.ID", create_if_not_exist=True)
dataset.upload_folder(Path("air_temperature.zarr"))