io.file file and directory access

Author: Chris Zheng  (z@caudate.me)
Date: 27 November 2018
Repository: https://github.com/zcaudate/hara
Version: 3.0.2

1    Introduction

hara.io.file aims to provide consistency of reporting for file operations. Operations are designed using the java.nio.file.FileVisitor pattern. File and directory manipulation are considered as bulk operations by default.

1.1    Installation

Add to project.clj dependencies:

[hara/io.file "3.0.2"]

All functions are in the hara.io.file namespace.

 (use (quote hara.io.file))

2    Operation



copy ^

copies all specified files from one to another

v 3.0
(defn copy
  ([source target]
   (copy source target {}))
  ([source target opts]
   (let [copy-fn (fn [{:keys [root path attrs target accumulator simulate]}]
                   (let [rel   (.relativize ^Path root path)
                         dest  (.resolve ^Path target rel)
                         copts (->> [:copy-attributes :nofollow-links]
                                    (or (:options opts))
                                    (mapv option/option)
                                    (into-array CopyOption))]
                     (when-not simulate
                       (Files/createDirectories (.getParent dest) attr/*empty*)
                       (Files/copy ^Path path ^Path dest copts))
                     (swap! accumulator
                            assoc
                            (str path)
                            (str dest))))]
     (walk/walk source
                (merge {:target (path/path target)
                        :directory copy-fn
                        :file copy-fn
                        :with #{:root}
                        :accumulator (atom {})
                        :accumulate #{}}
                       opts)))))
link
(copy "src" ".src" {:include [".clj"]}) => map? (delete ".src")

copy-single ^

copies a single file to a destination

v 3.0
(defn copy-single
  ([source target]
   (copy-single source target {}))
  ([source target opts]
   (if-let [dir (parent target)]
     (if-not (exists? dir)
       (create-directory dir)))
   (Files/copy ^Path (path/path source)
               ^Path (path/path target)
               (->> (:options opts)
                    (mapv option/option)
                    (into-array CopyOption)))))
link
(copy-single "project.clj" "dev/scratch/project.clj.bak" {:options #{:replace-existing}}) => (path "." "dev/scratch/project.clj.bak") (delete "dev/scratch/project.clj.bak")

create-directory ^

creates a directory on the filesystem

v 3.0
(defn create-directory
  ([path]
   (create-directory path {}))
  ([path attrs]
   (Files/createDirectories (path/path path)
                            (attr/map->attr-array attrs))))
link
(do (create-directory "dev/scratch/.hello/.world/.foo") (directory? "dev/scratch/.hello/.world/.foo")) => true (delete "dev/scratch/.hello")

create-symlink ^

creates a symlink to another file

v 3.0
(defn create-symlink
  ([path link-to]
   (create-symlink path link-to {}))
  ([path link-to attrs]
   (Files/createSymbolicLink (path/path path)
                             (path/path link-to)
                             (attr/map->attr-array attrs))))
link
(do (create-symlink "dev/scratch/project.lnk" "project.clj") (link? "dev/scratch/project.lnk")) => true

create-tmpdir ^

creates a temp directory on the filesystem

v 3.0
(defn create-tmpdir
  ([]
   (create-tmpdir ""))
  ([prefix]
   (Files/createTempDirectory prefix (make-array FileAttribute 0))))
link
(create-tmpdir) ;;=> #path:"/var/folders/d6/yrjldmsd4jd1h0nm970wmzl40000gn/T/4870108199331749225"

create-tmpfile ^

creates a tempory file

v 3.0
(defn create-tmpfile
  ([]
   (java.io.File/createTempFile "tmp" "")))
link
(create-tmpfile) ;;#file:"/var/folders/rc/4nxjl26j50gffnkgm65ll8gr0000gp/T/tmp2270822955686495575" => java.io.File

delete ^

copies all specified files from one to another

v 3.0
(defn delete
  ([root] (delete root {}))
  ([root opts]
   (let [delete-fn (fn [{:keys [path attrs accumulator simulate]}]
                     (try (if-not simulate
                            (Files/delete path))
                          (swap! accumulator conj (str path))
                          (catch DirectoryNotEmptyException e)))]
     (walk/walk root
                (merge {:directory {:post delete-fn}
                        :file delete-fn
                        :with #{:root}
                        :accumulator (atom #{})
                        :accumulate #{}}
                       opts)))))
link
(do (copy "src/hara/test.clj" ".src/hara/test.clj") (delete ".src" {:include ["test.clj"]})) => #{(str (path ".src/hara/test.clj"))} (delete ".src") => set?

list ^

lists the files and attributes for a given directory

v 3.0
(defn list
  ([root] (list root {}))
  ([root opts]
   (let [gather-fn (fn [{:keys [path attrs accumulator]}]
                     (swap! accumulator
                            assoc
                            (str path)
                            (str (permissions path) "/" (shorthand path))))]
     (walk/walk root
                (merge {:depth 1
                        :directory gather-fn
                        :file gather-fn
                        :accumulator (atom {})
                        :accumulate #{}
                        :with #{}}
                       opts)))))
link
(list "src") => {"/Users/chris/Development/chit/hara/src" "rwxr-xr-x/d", "/Users/chris/Development/chit/hara/src/hara" "rwxr-xr-x/d"} (list "../hara/src/hara/io" {:recursive true}) => {"/Users/chris/Development/chit/hara/src/hara/io" "rwxr-xr-x/d", "/Users/chris/Development/chit/hara/src/hara/io/file/reader.clj" "rw-r--r--/-", "/Users/chris/Development/chit/hara/src/hara/io/project.clj" "rw-r--r--/-", "/Users/chris/Development/chit/hara/src/hara/io/file/filter.clj" "rw-r--r--/-", ... ... "/Users/chris/Development/chit/hara/src/hara/io/file/path.clj" "rw-r--r--/-", "/Users/chris/Development/chit/hara/src/hara/io/file/walk.clj" "rw-r--r--/-", "/Users/chris/Development/chit/hara/src/hara/io/file.clj" "rw-r--r--/-"}

move ^

moves a file or directory

v 3.0
(defn move
  ([source target]
   (move source target {}))
  ([source target opts]
   (let [move-fn (fn [{:keys [root path attrs target accumulator simulate]}]
                   (let [rel   (.relativize ^Path root path)
                         dest  (.resolve ^Path target rel)
                         copts (->> [:atomic-move]
                                    (or (:options opts))
                                    (mapv option/option)
                                    (into-array CopyOption))]
                     (when-not simulate
                       (Files/createDirectories (.getParent dest) attr/*empty*)
                       (Files/move ^Path path ^Path dest copts))
                     (swap! accumulator
                            assoc
                            (str path)
                            (str dest))))
         results (walk/walk source
                            (merge {:target (path/path target)
                                    :recursive true
                                    :directory {:post (fn [{:keys [path]}]
                                                        (if (empty-directory? path)
                                                          (delete path opts)))}
                                    :file move-fn
                                    :with #{:root}
                                    :accumulator (atom {})
                                    :accumulate #{}}
                                   opts))]
     results)))
link
(do (move "shortlist" ".shortlist") (move ".shortlist" "shortlist")) (move ".non-existent" ".moved") => {}

select ^

selects all the files in a directory

v 3.0
(defn select
  ([root]
   (select root nil))
  ([root opts]
   (walk/walk root opts)))
link
(->> (select "src/hara/io/file") (map #(relativize "src/hara" %)) (map str) (sort)) => ["io/file" "io/file/attribute.clj" "io/file/charset.clj" "io/file/common.clj" "io/file/filter.clj" "io/file/option.clj" "io/file/path.clj" "io/file/reader.clj" "io/file/walk.clj" "io/file/watch.clj" "io/file/writer.clj"]

3    Path



file-name ^

returns the last section of the path

v 3.0
(defn file-name
  [x]
  (.getFileName (path x)))
link
(str (file-name "src/hara")) => "hara"

nth-segment ^

returns the nth segment of a given path

v 3.0
(defn nth-segment
  [x i]
  (.getName (path x) i))
link
(str (nth-segment "/usr/local/bin" 1)) => "local"

parent ^

returns the parent of the path

v 3.0
(defn parent
  [path]
  (.getParent (path/path path)))
link
(str (parent "/hello/world.html")) => "/hello"

path ^

creates a `java.nio.file.Path object

v 3.0
(defn path
  ([x]
   (cond (instance? Path x)
         x

         (string? x)
         (.normalize (Paths/get (normalise x) *empty-string-array*))

         (vector? x)
         (apply path x)

         (instance? java.net.URI x)
         (Paths/get x)

         (instance? File x)
         (path (.toString ^File x))

         :else
         (throw (Exception. (format "Input %s is not of the correct format" x)))))
  ([s & more]
   (.normalize (Paths/get (normalise (str s)) (into-array String (map str more))))))
link
(path "project.clj") ;;=> #path:"/Users/chris/Development/chit/hara/project.clj" (path (path "project.clj")) ;; idempotent ;;=> #path:"/Users/chris/Development/chit/hara/project.clj" (path "~") ;; tilda ;;=> #path:"/Users/chris" (path "src" "hara/time.clj") ;; multiple arguments ;;=> #path:"/Users/chris/Development/chit/hara/src/hara/time.clj" (path ["src" "hara" "time.clj"]) ;; vector ;;=> #path:"/Users/chris/Development/chit/hara/src/hara/time.clj" (path (java.io.File. ;; java.io.File object "src/hara/time.clj")) ;;=> #path:"/Users/chris/Development/chit/hara/src/hara/time.clj" (path (java.net.URI. ;; java.net.URI object "file:///Users/chris/Development/chit/hara/project.clj")) ;;=> #path:"/Users/chris/Development/chit/hara/project.clj"

path? ^

checks to see if the object is of type Path

v 3.0
(defn path?
  [x]
  (instance? Path x))
link
(path? (path "/home")) => true

relativize ^

returns the relationship between two paths

v 3.0
(defn relativize
  [path1 path2]
  (.relativize (path/path path1) (path/path path2)))
link
(str (relativize "hello" "hello/world.html")) => "world.html"

root ^

returns the root path

v 3.0
(defn root
  [x]
  (.getRoot (path x)))
link
(str (root "/usr/local/bin")) => "/"

segment-count ^

returns the number of segments of a given path

v 3.0
(defn segment-count
  [x]
  (.getNameCount (path x)))
link
(segment-count "/usr/local/bin") => 3

subpath ^

returns the subpath of a given path

v 3.0
(defn subpath
  [x start end]
  (.subpath (path x) start end))
link
(str (subpath "/usr/local/bin/hello" 1 3)) => "local/bin"

4    File



file ^

returns the input as a file

v 3.0
(defn file
  [path]
  (path/to-file (path/path path)))
link
(file "project.clj") => java.io.File

file->ns ^

converts a file string to an ns string

v 3.0
(defn file->ns
  [ns]
  (-> ns
      (.replaceAll "/" ".")
      (.replaceAll "_" "-")))
link
(file->ns "hara/io/file_test") => "hara.io.file-test"

file-system ^

returns the filesystem governing the path

v 3.0
(defn file-system
  [x]
  (.getFileSystem (path x)))
link
(file-system ".") ;; #object[sun.nio.fs.MacOSXFileSystem 0x512a9870 "sun.nio.fs.MacOSXFileSystem@512a9870"] => java.nio.file.FileSystem

input-stream ^

opens a file as an input-stream

v 3.0
(defn input-stream
  ([path]
   (input-stream path {}))
  ([path opts]
   (Files/newInputStream (path/path path)
                         (->> (:options opts)
                              (mapv option/option)
                              (into-array OpenOption)))))
link
(input-stream "project.clj")

input-stream? ^

checks if object is an input-stream

v 3.0
(defn input-stream?
  ([x]
   (instance? java.io.InputStream x)))
link
(input-stream? (input-stream "project.clj"))

ns->file ^

converts an ns string to a file string

v 3.0
(defn ns->file
  [ns]
  (-> (str ns)
      (.replaceAll "\." "/")
      (.replaceAll "-" "_")))
link
(ns->file 'hara.io.file-test) => "hara/io/file_test"

option ^

shows all options for file operations

v 3.0
(defn option
  ([] (keys +all-options+))
  ([k]
   (+all-options+ k)))
link
(option) => (contains [:atomic-move :create-new :skip-siblings :read :continue :create :terminate :copy-attributes :append :truncate-existing :sync :follow-links :delete-on-close :write :dsync :replace-existing :sparse :nofollow-links :skip-subtree]) (option :read) => java.nio.file.StandardOpenOption/READ

output-stream ^

opens a file as an output-stream

v 3.0
(defn output-stream
  ([path]
   (output-stream path {}))
  ([path opts]
   (Files/newOutputStream (path/path path)
                          (->> (:options opts)
                               (mapv option/option)
                               (into-array OpenOption)))))
link
(output-stream "project.clj")

output-stream? ^

checks if object is an output-stream

v 3.0
(defn output-stream?
  ([x]
   (instance? java.io.OutputStream x)))
link
(output-stream? (output-stream "project.clj"))

read-all-bytes ^

opens a file and reads the contents as a byte array

v 3.0
(defn read-all-bytes
  [path]
  (Files/readAllBytes (path/path path)))
link
(read-all-bytes "project.clj")

read-all-lines ^

opens a file and reads the contents as an array of lines

v 3.0
(defn read-all-lines
  [path]
  (Files/readAllLines (path/path path)))
link
(read-all-lines "project.clj")

read-code ^

takes a file and returns a lazy seq of top-level forms

v 3.0
(defn read-code
  [path]
  (with-open [reader (reader/reader :pushback path)]
    (->> (repeatedly #(try (read reader)
                           (catch Throwable e)))
         (take-while identity)
         (doall))))
link
(->> (read-code "src/hara/io/file.clj") first (take 2)) => '(ns hara.io.file)

resource ^

accesses a url on the classpath

v 3.0
(defn ^java.net.URL resource
  ([n] (resource n (.getContextClassLoader (Thread/currentThread))))
  ([n ^ClassLoader loader] (.getResource loader n)))
link
(resource "project.clj") => java.net.URL

write ^

writes a stream to a path

v 3.0
(defn write
  ([stream path]
   (write stream path {}))
  ([stream path opts]
   (Files/copy stream
               ^Path (path/path path)
               (->> (:options opts)
                    (mapv option/option)
                    (into-array CopyOption)))))
link
(-> (java.io.FileInputStream. "project.clj") (write "project.clj" {:options #{:replace-existing}}))

write-all-bytes ^

writes a byte-array to file

v 3.0
(defn write-all-bytes
  ([path bytes]
   (write-all-bytes path bytes {}))
  ([path bytes opts]
   (Files/write (path/path path)
                bytes
                (->> (:options opts)
                     (mapv option/option)
                     (into-array OpenOption)))))
link
(write-all-bytes "hello.txt" (.getBytes "Hello World"))

5    Attribute



attributes ^

shows all attributes for a given path

v 3.0
(defn attributes
  [path]
  (-> (path/path path)
      (Files/readAttributes (str (name common/*system*) ":*")
                            common/*no-follow*)
      (attrs->map)))
link
(attributes "project.clj") ;; {:owner "chris", ;; :group "staff", ;; :permissions "rw-r--r--", ;; :file-key "(dev=1000004,ino=2351455)", ;; :ino 2351455, ;; :is-regular-file true. ;; :is-directory false, :uid 501, ;; :is-other false, :mode 33188, :size 4342, ;; :gid 20, :ctime 1476755481000, ;; :nlink 1, ;; :last-access-time 1476755481000, ;; :is-symbolic-link false, ;; :last-modified-time 1476755481000, ;; :creation-time 1472282953000, ;; :dev 16777220, :rdev 0} => map

directory? ^

checks whether a file is a directory

v 3.0
(defn directory?
  [path]
  (Files/isDirectory (path/path path) common/*no-follow*))
link
(directory? "src") => true (directory? "project.clj") => false

empty-directory? ^

checks if a directory is empty, returns true if both are true

v 3.0
(defn empty-directory?
  [path]
  (if (directory? path)
    (= 1 (count (list path)))
    (throw (Exception. (str "Not a directory: " path)))))
link
(empty-directory? ".") => false

executable? ^

checks whether a file is executable

v 3.0
(defn executable?
  [path]
  (Files/isExecutable (path/path path)))
link
(executable? "project.clj") => false (executable? "/usr/bin/whoami") => true

exists? ^

checks whether a file exists

v 3.0
(defn exists?
  [path]
  (Files/exists (path/path path) common/*no-follow*))
link
(exists? "project.clj") => true (exists? "NON.EXISTENT") => false

file-type ^

encodes the type of file as a keyword

v 3.0
(defn file-type
  [file]
  (-> (str file)
      (string/split #".")
      last
      keyword))
link
(file-type "hello.clj") => :clj (file-type "hello.java") => :java

file? ^

checks whether a file is not a link or directory

v 3.0
(defn file?
  [path]
  (Files/isRegularFile (path/path path) common/*no-follow*))
link
(file? "project.clj") => true (file? "src") => false

hidden? ^

checks whether a file is hidden

v 3.0
(defn hidden?
  [path]
  (Files/isHidden (path/path path)))
link
(hidden? ".gitignore") => true (hidden? "project.clj") => false

link? ^

checks whether a file is a link

v 3.0
(defn link?
  [path]
  (Files/isSymbolicLink (path/path path)))
link
(link? "project.clj") => false (link? (create-symlink "project.lnk" "project.clj")) => true (delete "project.lnk")

permissions ^

returns the permissions for a given file

v 3.0
(defn permissions
  [path]
  (let [path (path/path path)]
    (->> common/*no-follow*
         (Files/getPosixFilePermissions path)
         (PosixFilePermissions/toString))))
link
(permissions "src") => "rwxr-xr-x"

readable? ^

checks whether a file is readable

v 3.0
(defn readable?
  [path]
  (Files/isReadable (path/path path)))
link
(readable? "project.clj") => true

set-attributes ^

sets all attributes for a given path

v 3.0
(defn set-attributes
  [path m]
  (reduce-kv (fn [_ k v]
               (-> (path/path path)
                   (Files/setAttribute (str (name common/*system*) ":"
                                            (string/camel-case (name k)))
                                       (attr-value k v)
                                       common/*no-follow*)))
             nil
             m))
link
(set-attributes "project.clj" {:owner "chris", :group "staff", :permissions "rw-rw-rw-"}) ;;=> #path:"/Users/chris/Development/chit/lucidity/project.clj"

shorthand ^

returns the shorthand string for a given entry

v 3.0
(defn shorthand
  [path]
  (let [path (path/path path)]
    (cond (Files/isDirectory path (LinkOption/values))
          "d"

          (Files/isSymbolicLink path)
          "l"

          :else "-")))
link
(shorthand "src") => "d" (shorthand "project.clj") => "-"

writable? ^

checks whether a file is writable

v 3.0
(defn writable?
  [path]
  (Files/isWritable (path/path path)))
link
(writable? "project.clj") => true

6    Advanced

As all bulk operations are based on hara.io.file.walk/walk, it provides a consistent interface for working with files:

6.1    depth

The :depth option determines how far down the directory listing to move

;; listing the src directory to a depth of 2

(list "src" {:depth 2})
=> {"/Users/chris/Development/chit/hara/src" "rwxr--r--/d",
    "/Users/chris/Development/chit/hara/src/hara/data.clj" "rw-r--r--/-",
    ... ...
    "/Users/chris/Development/chit/hara/src/hara.function" "rwxr-xr-x/d"}

;; copying the src directory to a depth of 2
(copy "src" ".src" {:depth 2})

;; delete the .src directory to a depth of 2
(delete ".src" {:depth 2})

The :recursive flag enables walking to all depths

;; lists contents of src directory, :recursive is false by default
(list "src" {:recursive true})
=> {"/Users/chris/Development/chit/hara/src" "rwxr--r--/d",
    "/Users/chris/Development/chit/hara/src/hara/data.clj" "rw-r--r--/-",
    ... ...
    "/Users/chris/Development/chit/hara/src/hara.function" "rwxr-xr-x/d"}

;; copying the src directory, :recursive is true by default
(copy "src" ".src" {:recursive false})
=> {"src" ".src",
    "src/hara" ".src/hara"}

;; delete the .src directory, :recursive is true by default
(delete ".src" {:recursive false})
=> #{"/Users/chris/Development/chit/hara/.src/hara"
     "/Users/chris/Development/chit/hara/.src"}

6.2    simulate

When the :simulate flag is set, the operation is not performed but will output as if the operation has been done.

(copy "src" ".src" {:simulate true})
=> {"src" ".src",
    "src/hara" ".src/hara"}

(move "src" ".src" {:simulate true})
=> {"/Users/chris/Development/chit/hara/src/hara/data.clj"
    "/Users/chris/Development/chit/hara/.src/hara/data.clj",
    ... ...
    "/Users/chris/Development/chit/hara/src/hara/time/data/coerce.clj"
    "/Users/chris/Development/chit/hara/.src/hara/time/data/coerce.clj"}

(delete ".src" {:simulate true})
=> #{"/Users/chris/Development/chit/hara/.src/hara"
     "/Users/chris/Development/chit/hara/.src"}

6.3    filter

Files can be included or excluded through an array of file filters. Values for :exclude and :include array elements can be either a pattern or a function:

(select "." {:exclude [".clj"
                       directory?]
             :recursive false})
=> [;; #path:"/Users/chris/Development/chit/hara/.gitignore"
    ;; #path:"/Users/chris/Development/chit/hara/.gitmodules"
    ...
    ;; #path:"/Users/chris/Development/chit/hara/spring.jpg"
    ;; #path:"/Users/chris/Development/chit/hara/travis.jpg"
]

(select "." {:include [".clj$"
                       file?]
             :recursive false})
=> [;; #path:"/Users/chris/Development/chit/hara/.midje.clj"
    ;; #path:"/Users/chris/Development/chit/hara/project.clj"
]

:accumulate is another options to set that specifies which files are going to be included in in accumulation:

(select "." {:accumulate #{}})
=> []

(select "src" {:depth 2
               :accumulate #{:directories}})
=> [;; #path:"/Users/chris/Development/chit/hara/src"
    ;; #path:"/Users/chris/Development/chit/hara/src/hara"
]

(select "src" {:depth 2
               :accumulate #{:files}})

=> [;; #path:"/Users/chris/Development/chit/hara/src/hara.core.base.class.clj"
    ...
    ;; #path:"/Users/chris/Development/chit/hara/src/hara/time.clj"
    ;; #path:"/Users/chris/Development/chit/hara/src/hara/zip.clj"
]

6.4    file system

The :file option takes a function which will run whenever a file is visited:

(select "src" {:include ["/class/"]
               :file (fn [{:keys [path]}] (println (str path)))})
;; /Users/chris/Development/chit/hara/src/hara.core.base.class.clj
;; /Users/chris/Development/chit/hara/src/hara.core.base.class/enum.clj
;; /Users/chris/Development/chit/hara/src/hara.core.base.class/inheritance.clj
;; /Users/chris/Development/chit/hara/src/hara.core.base.class/multi.clj

The :directory option takes either a function or a map of function which will run whenever a directory is visited:

(select "src" {:include ["/class"]
               :directory (fn [{:keys [path]}]
                            (println (str path)))})

;; /Users/chris/Development/chit/hara/src/hara.core.base.class

(select "src" {:include ["/class"]
               :directory {:pre  (fn [{:keys [path]}]
                                   (println "PRE" (str path)))
                           :post (fn [{:keys [path]}]
                                   (println "POST " (str path)))}})

;; PRE /Users/chris/Development/chit/hara/src/hara.core.base.class
;; POST  /Users/chris/Development/chit/hara/src/hara.core.base.class

:options are passed in for move and copy

;; `:replace-existing` replaces an existing file if it exists.
;; `:copy-attributes`  copy attributes to the new file.

(copy "project.clj" "project.clj.bak"
      {:options [:replace-existing
                 :copy-attributes]})

;; `:atomic-move` moves the file as an atomic file system operation.

(move "project.clj.bak" "project.clj"
      {:options [:replace-existing
                 :atomic-move]})

:with is either #{:root} to include the root path or #{} to not include the root path. It is set to #{:root} for copy, move and delete. It is set to #{} for list and select.

6.5    accumulator

:accumulator sets the atom that contains the accumulated values during the walk:

(let [acc (atom [])]

  (select "src/hara.core.base.class"  {:accumulator acc})

  (select "src/hara/common" {:accumulator acc})

  (map #(str (relativize "src/hara" %))
       @acc))
=> ("class"
    "class/checks.clj"
    "class/enum.clj"
    "class/inheritance.clj"
    "class/multi.clj"
    "common"
    "common/checks.clj"
    "common/error.clj"
    "common/hash.clj"
    "common/pretty.clj"
    "common/primitives.clj"
    "common/state.clj"
    "common/string.clj"
    "common/watch.clj")

For more examples of how it is used, please see the source code for copy, delete list and move.