Get S3 filesystem details using PySpark

Unfortunately Spark doesn’t give you information about underlying files - the deepest function to get any info is input_file_name but that’s it. But you can do that using HDFS API, here is a function I wrote

from pyspark.sql import DataFrame, SparkSession, Column

def list_files_with_hdfs(spark: SparkSession, path_root: str):
    sc = spark.sparkContext
    java_path = sc._jvm.java.net.URI.create(path_root)
    hadoop_path = sc._jvm.org.apache.hadoop.fs.Path(path_root)
    hadoop_file_system = sc._jvm.org.apache.hadoop.fs.FileSystem.get(
        java_path,
        sc._jvm.org.apache.hadoop.conf.Configuration())
    iterator = hadoop_file_system.listFiles(hadoop_path, True)

    result = []
    while iterator.hasNext():
        item = iterator.next()
        el = {
            "path": item.getPath().toString(),
            "length": item.getLen(),
            "modTime": item.getModificationTime()
        }
        result.append(el)
        if len(result) % 1000 == 0:
          print(f"{len(result)}. ", end="")

    return result

the modTime is a long number, so you can prettify it as well:

fis = list_files_with_hdfs(spark, "s3://a_bucket/")

df = spark.createDataFrame(fis)

df = (df
     .select(df.path,
             df.length.alias("lengthBytes"),
             (df.modTime / 1000).cast("timestamp").alias("modTime") ))

display(df)

Thanks! You can always email me or use contact form for more questions/comments etc.