Connect & read Datalake from Python

A couple Azure libraries need to be installed first.

#pip install azure-storage-file-datalake

#pip install azure-identity

If using service principal:

from azure.storage.filedatalake import DataLakeServiceClient

from azure.identity import ClientSecretCredential

tenant_id=""

app_id=""

secret=""

storage_account_name = ""

container_name = ''

account_url = 'https://{}.dfs.core.windows.net/'.format(storage_account_name)

credential = ClientSecretCredential(tenant_id, app_id, secret)

service = DataLakeServiceClient(account_url=account_url, credential=credential)

file_system_client = service.get_file_system_client(container_name)

for path in file_system_client.get_paths('/'):

print(path.name)

If using access key:

from azure.storage.filedatalake import DataLakeServiceClient

container_name = ''

account_url = 'https://{}.dfs.core.windows.net/'.format(storage_account_name)

access_key = ''

datalake_service_client = DataLakeServiceClient(account_url=account_url, credential=access_key)

file_systems = datalake_service_client.list_file_systems()

for file_system in file_systems:

    print(file_system.name)

Note different service principal / access keys may have different access to the data lake. So some of the operations may not work, e.g. list file systems( i.e. containers) , but other ops, e.g. get paths for a specific container may work.

To download files from a subdirectory using access key

Connect to the file system and list all files under a directory. Then download each of the files into one file on the local. Go from there, e.g. load file to pandas, etc.

#https://docs.microsoft.com/en-us/azure/architecture/data-science-process/explore-data-blob

from azure.storage.filedatalake import DataLakeServiceClient

endpoint = 'https://mydatalake001.dfs.core.windows.net/'

access_key = 'xxxxxxx'

service_client = DataLakeServiceClient(account_url=endpoint, credential=access_key)

container = 'mycontianer'

site = 'xxx'

partition = '2022-07-26'

path=f"osipi/archive/{site}/csv/{partition}"

local_temp_file = 'c:\\temp\\download.csv'

try:    

    file_system_client = service_client.get_file_system_client(file_system=container )

    directory_client = file_system_client.get_directory_client(path)

    paths = file_system_client.get_paths(path) #get all files

    with open(local_temp_file, 'wb') as local_file:

        for path in paths:

            #print('downloading {}'.format(path.name))          

            file_client = directory_client.get_file_client(path)  

            download = file_client.download_file()

            downloaded_bytes = download.readall()

            local_file.write(downloaded_bytes)

except Exception as e:

    print(e)

To download files from subdirectory using service principal

#pip install azure-storage-file-datalake

#pip install azure-identity

from azure.storage.filedatalake import DataLakeServiceClient

from azure.identity import ClientSecretCredential

import pandas as pd

tenant_id="xxx"

app_id="xxxe"

secret="xxx"

storage_account_name = "xxx"

account_url = 'https://{}.dfs.core.windows.net/'.format(storage_account_name)

credential = ClientSecretCredential(tenant_id, app_id, secret)

service = DataLakeServiceClient(account_url=account_url, credential=credential)

container_name = 'containername'

path = 'dir1/dir2'

local_temp_file = 'c:\\temp\\download.csv'

try:    

    file_system_client = service.get_file_system_client(container_name)

    directory_client = file_system_client.get_directory_client(path)

    paths = file_system_client.get_paths(path) #get all files

    with open(local_temp_file, 'wb') as local_file:

        for path in paths:

            print('downloading {}'.format(path.name))       

            file_client = directory_client.get_file_client(path)  

            download = file_client.download_file()

            downloaded_bytes = download.readall()

            local_file.write(downloaded_bytes)

        df = pd.read_csv(local_temp_file)

        df.head()

except Exception as e:

    print(e)