Changeset 15397


Ignore:
Timestamp:
10/10/20 21:43:31 (10 months ago)
Author:
Mark Evenson
Message:

Expand abstraction of Archive

Fix ZipTest?

Start working on creating a test for a nested jar.

pathname: overhaul semantics for zips using PATHNAME-DEVICE

Implement CL:PATHNAME-DEVICE references to files which are cons cells
to the level of abcl-1.7.0, which is that nested pathnames do not
fully work for all variants of CL:TRUENAME.

Attempt to start coding "worst case, cache everything we find
strategy".

TODO memoize storage via a weak reference so that the VM can possibly
reclaim space if it needs to.

TODO re-implement references to URIs. (???)

  • * *

OPEN of remote Zip archives works

TODO no last modified times are being returned

Location:
trunk/abcl
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/abcl/src/org/armedbear/lisp/PathnameJar.java

    r15396 r15397  
    339339  }
    340340
     341  String getRootJarAsURLString() {
     342   return
     343     PathnameJar.JAR_URI_PREFIX
     344     + ((Pathname)getRootJar()).getNamestring()
     345     + PathnameJar.JAR_URI_SUFFIX;
     346  }
     347
     348
    341349  LispObject getJars() {
    342350    return getDevice();
    … …  
    351359    LispObject enclosingJars = jars.cdr();
    352360
    353     if (!rootJar.isLocalFile()) {
    354       // FIXME implement me
    355       simple_error("Unimplemented TRUENAME for non-file root jar.");
    356     }
     361    // if (!rootJar.isLocalFile()) {
     362       
     363    // }
    357364
    358365    PathnameJar p = new PathnameJar();
    … …  
    363370      ZipCache.Archive archive = ZipCache.getArchive(p);
    364371      if (archive == null) {
    365         if (errorIfDoesNotExist) {
    366           return simple_error("Accessible TRUENAME can't be determined for: ~a", pathname); //time !?
    367         } else {
    368           return NIL;
    369         }
     372        return Pathname.doTruenameExit(pathname, errorIfDoesNotExist);
    370373      }
    371374      return p;
    … …  
    374377    ZipEntry entry = ZipCache.getZipEntry(p);
    375378    if (entry == null) {
    376       if (errorIfDoesNotExist) {
    377         return simple_error("Accessible TRUENAME can't be determined for: ~a", pathname); //time !?
    378       } else {
    379         return NIL;
    380       }
     379      return Pathname.doTruenameExit(pathname, errorIfDoesNotExist);
    381380    }
    382381    return p;

  • trunk/abcl/src/org/armedbear/lisp/ZipCache.java

    r15395 r15397  
    5555import java.util.Map;
    5656import java.util.Set;
     57import java.util.logging.Level;
     58import java.util.logging.Logger;
    5759import java.util.zip.ZipException;
    5860import java.util.zip.ZipFile;
    … …  
    174176    } else {
    175177      Archive archive = ZipCache.getArchive(archiveEntry);
    176       ZipFile zipFile = archive.file;
    177       ZipEntry entry = archive.getEntry(archiveEntry);
     178      // ZipFile zipFile = archive.file;
     179      // ZipEntry entry = archive.getEntry(archiveEntry);
    178180     
    179       try {
    180         result = zipFile.getInputStream(entry);
    181       } catch (IOException e) {
     181      result = archive.getEntryAsInputStream(archiveEntry);
     182      if (result == null) {
    182183        simple_error("Failed to get InputStream for ~a", archiveEntry);
    183184      }
    … …  
    195196  static HashMap<PathnameJar, Archive> cache = new HashMap<PathnameJar, Archive>();
    196197
    197   static public class Archive {
    198     ZipFile file;
    199     ZipInputStream inputStream;   // Unused, speculative
     198  abstract static public class Archive {
     199    PathnameJar root;
    200200    LinkedHashMap<PathnameJar, ZipEntry> entries
    201201      = new LinkedHashMap<PathnameJar, ZipEntry>();
    202202    long lastModified;
     203
     204    abstract InputStream getEntryAsInputStream(PathnameJar entry);
     205    abstract ZipEntry getEntry(PathnameJar entry);
     206    abstract void populateAllEntries();
     207    abstract void close();
     208  }
     209
     210  static public class ArchiveStream
     211    extends Archive
     212  {
     213    InputStream source;
     214
     215    public InputStream getEntryAsInputStream(PathnameJar entry) {
     216      return null;
     217    }
     218
     219    public ZipEntry getEntry(PathnameJar entry) {
     220      return null;
     221    }
     222
     223    void populateAllEntries() {}
     224
     225    void close () {
     226      if (source != null) {
     227        try {
     228          source.close();
     229        } catch (IOException ex) {
     230          {}
     231        }
     232      }
     233    }
     234  }
     235
     236  static public class ArchiveURL
     237    extends ArchiveFile
     238  {
     239    JarURLConnection connection;
     240    void close() {
     241      super.close();
     242      // TODO: do we need to clean up from the connection?
     243    }
     244  }
     245
     246  static public class ArchiveFile
     247    extends Archive
     248  {
     249    ZipFile file;
     250
     251    ZipFile get() { return file;}
    203252
    204253    public ZipEntry getEntry(PathnameJar entryPathname) {
    … …  
    226275    }
    227276
    228     void populateAllEntries(PathnameJar jar) {
     277    void populateAllEntries() {
    229278      ZipFile f = file;
    230279      if (f.size() == entries.size()) {
    … …  
    237286        String name = entry.getName();
    238287        PathnameJar entryPathname
    239           = (PathnameJar)PathnameJar.createEntryFromJar(jar, name);
     288          = (PathnameJar)PathnameJar.createEntryFromJar(root, name);
    240289        entries.put(entryPathname, entry);
    241290      }
    242291    }
    243   }
     292
     293    InputStream getEntryAsInputStream(PathnameJar entry) {
     294      InputStream result = null;
     295      ZipEntry zipEntry = getEntry(entry);
     296
     297      try {
     298        result = file.getInputStream(zipEntry);
     299      } catch (IOException e) {}
     300
     301      return result;
     302    }
     303    void close() {
     304      if (file != null) {
     305        try {
     306          file.close();
     307        } catch (IOException e) {}
     308         
     309      }
     310    }
     311  }
     312 
    244313
    245314  static boolean cacheEnabled = true;
    … …  
    263332  synchronized public static LinkedHashMap<PathnameJar,ZipEntry> getEntries(PathnameJar jar) {
    264333    Archive archive = getArchive(jar);
    265     archive.populateAllEntries(jar); // Very expensive for jars with large number of entries
     334    archive.populateAllEntries(); // Very expensive for jars with large number of entries
    266335    return archive.entries;
    267336  }
    … …  
    282351
    283352  synchronized public static Archive getArchive(PathnameJar jar) {
     353    Archive cached = cache.get(jar);
     354    if (cached != null) {
     355      return cached;
     356    }
     357
    284358    Pathname rootJar = (Pathname) jar.getRootJar();
    285359    LispObject innerJars = jar.getJars().cdr();
    286     if (innerJars != NIL) {
     360
     361    if (!rootJar.isLocalFile()) {
     362      return getArchiveURL(jar);
     363    }
     364   
     365    if (innerJars.equals(NIL)) {
     366      return getArchiveFile(jar);
     367    } else {
     368      PathnameJar rootPathname = (PathnameJar)PathnameJar.createFromPathname(rootJar);
     369
     370      Archive current = ZipCache.getArchive(rootPathname);
     371     
     372      while (innerJars.car() != NIL) {
     373      // TODO Finish me!
     374      // Pathname next = (Pathname)innerJars.car();
     375      // String entryPath = next.asEntryPath();
     376      // InputStream source;
     377      // if (current instanceof ArchiveFile) {
     378      //   ArchiveFile zipFile = ((ArchiveFile)current).get();
     379      //   ZipEntry entry = zipFile.getEntry(entryPath);
     380      //   source = zipFile.getInputStream(entry);
     381      //   ArchiveStream stream = new ArchiveStream();
     382      //   stream.source = source;
     383      // }
     384      }
    287385      // FIXME
    288386      simple_error("Currently unimplemented recursive archive: ~a" , jar);
    289387      return (Archive)UNREACHED;
    290388    }
    291 
    292     if (jar.isLocalFile()) {
    293       Archive cached = cache.get(jar);
    294       if (cached != null) {
    295         return cached;
    296       }
    297 
    298       File f = rootJar.getFile();
    299      
    300       try {
    301         Archive result = new Archive();
    302         result.file = new ZipFile(f);
    303         result.lastModified = f.lastModified();
    304         cache.put(jar, result);
    305 
    306         return result;
    307       } catch (ZipException e) {
    308         error(new FileError("Failed to construct ZipFile"
    309                             + " because " + e,
    310                             Pathname.makePathname(f)));
    311         return null;
    312       } catch (IOException e) {
    313         error(new FileError("Failed to contruct ZipFile"
    314                             + " because " + e,
    315                             Pathname.makePathname(f)));
    316         return (Archive)UNREACHED;
    317       }
    318     } else {
    319       simple_error("Unimplemented fetch of remote resource.");
     389  }
     390
     391  public static Archive getArchiveURL(PathnameJar jar) {
     392    Pathname rootJar = (Pathname) jar.getRootJar();
     393    String rootJarURLString = jar.getRootJarAsURLString();
     394    URL rootJarURL = null;
     395    try {
     396      rootJarURL = new URL(rootJarURLString);
     397      JarURLConnection jarConnection
     398        = (JarURLConnection) rootJarURL.openConnection();
     399
     400      ArchiveURL result = new ArchiveURL();
     401      result.root = jar;
     402      result.connection = jarConnection;
     403      result.file = (ZipFile)jarConnection.getJarFile();
     404      result.lastModified = 0; // FIXME
     405      cache.put(jar, result);
     406      return result;
     407    } catch (MalformedURLException e) {
     408      simple_error("Failed to form root URL for ~a: ~a", jar, rootJarURLString);
     409       return (Archive)UNREACHED;     
     410    } catch (IOException e) {
     411      simple_error("Failed to fetch ~a: ~a", jar, e);
     412      return (Archive)UNREACHED;     
     413    }
     414  }
     415
     416  static public Archive getArchiveFile(PathnameJar jar) {
     417    try {
     418      ArchiveFile result = new ArchiveFile();
     419      File f = ((Pathname)jar.getRootJar()).getFile();
     420      result.root = jar;
     421      result.file = new ZipFile(f);
     422      result.lastModified = f.lastModified();
     423      cache.put(jar, result);
     424      return result;
     425    } catch (ZipException e) {
     426      error(new FileError("Failed to open local zip archive"
     427                          + " because " + e, jar));
     428                         
    320429      return (Archive)UNREACHED;
    321       // CachedItem e = fetchURL(jarOrEntry, false);
    322       // return e;
     430    } catch (IOException e) {
     431      error(new FileError("Failed to open local zip archive"
     432                          + " because " + e, jar));
     433      return (Archive)UNREACHED;
    323434    }
    324435  }
    … …  
    387498  // }
    388499
    389   static final SimpleDateFormat ASCTIME
    390     = new SimpleDateFormat("EEE MMM d HH:mm:ss yyyy", Locale.US);
    391   static final SimpleDateFormat RFC_1036
    392     = new SimpleDateFormat("EEEE, dd-MMM-yy HH:mm:ss zzz", Locale.US);
    393   static final SimpleDateFormat RFC_1123
    394     = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
    395 
    396   static void checkRemoteLastModified (PathnameJar jarEntry) {
     500
     501  static void checkRemoteLastModified(ArchiveURL archive) {
    397502    // Unfortunately, the Apple JDK under OS X doesn't do
    398503    // HTTP HEAD requests, instead refetching the entire
    … …  
    404509
    405510    String dateString = null;
    406     // try {
    407     //   dateString = HttpHead.get(url, "Last-Modified");
    408     // } catch (IOException ex) {
    409     //   Debug.trace(ex);
    410     // }
    411     // Date date = null;
    412     // ParsePosition pos = new ParsePosition(0);
     511
     512    String url = archive.root.getRootJarAsURLString();
    413513   
    414     // if (dateString != null) {
    415     //   date = RFC_1123.parse(dateString, pos);
    416     //   if (date == null) {
    417     //     date = RFC_1036.parse(dateString, pos);
    418     //     if (date == null)
    419     //       date = ASCTIME.parse(dateString, pos);
    420     //   }
    421     // }
    422      
    423     // if (date == null || date.getTime() > entry.lastModified) {
    424     //   entry = fetchURL(url, false);
    425     //   cache.put(url, entry);
    426     //   }
    427     //   if (date == null) {
    428     //     if (dateString == null) {
    429     //       Debug.trace("Failed to retrieve request header: "
    430     //                   + url.toString());
    431     //     } else {
    432     //       Debug.trace("Failed to parse Last-Modified date: " +
    433     //                   dateString);
    434     //     }
    435     //   }
     514    try {
     515      dateString = HttpHead.get(url, "Last-Modified");
     516    } catch (IOException ex) {
     517      Debug.trace(ex);
     518    }
     519    Date date = null;
     520    ParsePosition pos = new ParsePosition(0);
     521
     522    final SimpleDateFormat ASCTIME
     523      = new SimpleDateFormat("EEE MMM d HH:mm:ss yyyy", Locale.US);
     524    final SimpleDateFormat RFC_1036
     525      = new SimpleDateFormat("EEEE, dd-MMM-yy HH:mm:ss zzz", Locale.US);
     526    final SimpleDateFormat RFC_1123
     527      = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
     528
     529    if (dateString != null) {
     530      date = RFC_1123.parse(dateString, pos);
     531      if (date == null) {
     532        date = RFC_1036.parse(dateString, pos);
     533        if (date == null) {
     534          date = ASCTIME.parse(dateString, pos);
     535        }
     536      }
     537    }
     538
     539    // Replace older item in cache
     540    if (date == null || date.getTime() > archive.lastModified) {
     541      PathnameJar root = archive.root;
     542      Archive entry = getArchiveURL(root);
     543      cache.put(root, entry);
     544    }
     545    if (date == null) {
     546      if (dateString == null) {
     547        Debug.trace("Failed to retrieve request header: "
     548                    + url.toString());
     549      } else {
     550        Debug.trace("Failed to parse Last-Modified date: " +
     551                    dateString);
     552      }
     553    }
    436554  }
    437555
    438556  // FIXME recode once local fileaccesses are working
    439   static private ZipFile fetchURL(PathnameJar url, boolean useCaches) {
     557  static private ZipFile getRemote(PathnameJar url) {
    440558    ZipFile result = null;
    441559    URL jarURL = null;
    … …  
    461579    }
    462580    JarURLConnection jarURLConnection = (JarURLConnection) connection;
    463     jarURLConnection.setUseCaches(useCaches);
     581    //    jarURLConnection.setUseCaches(useCaches);
    464582    try {
    465583      result = jarURLConnection.getJarFile();
    … …  
    470588    }
    471589    if (cacheEnabled) {
    472       Archive archive = new Archive();
     590      ArchiveURL archive = new ArchiveURL();
    473591      archive.file = result;
    474592      archive.lastModified = jarURLConnection.getLastModified();
     593      archive.root = url;
    475594      cache.put(url, archive);
    476595    }
    … …  
    498617    Archive archive = cache.get(p.getNamestring());
    499618    if (archive != null) {
    500       try {
    501         if (archive.file != null) {
    502           archive.file.close();
    503         }
    504         if (archive.inputStream != null) {
    505           archive.inputStream.close();
    506         }
    507         cache.remove(p);
    508        
    509         return true;
    510       } catch (IOException e) {
    511         simple_error("failed to close zip cache references", e);
    512       }
     619      archive.close();
     620      cache.remove(p);
     621      return true;
    513622    }
    514623    return false;

  • trunk/abcl/src/org/armedbear/lisp/util/HttpHead.java

    r14627 r15397  
    5050 */
    5151public class HttpHead {
    52     static private String get(String urlString, String key) throws IOException {
     52    static public String get(String urlString, String key) throws IOException {
    5353        URL url = null;
    5454        try {

  • trunk/abcl/test/lisp/abcl/jar-pathname.lisp

    r15370 r15397  
    209209
    210210(defparameter *url-jar-pathname-base*
    211   "jar:http://abcl.org/fasl/42/baz-20140105a-fasl-42.jar!/")
    212 ;; fasl 41  "jar:http://abcl-dynamic-install.googlecode.com/files/baz-20130403a.jar!/")
    213 
    214 (defmacro load-url-relative (path)
    215   `(load (probe-file (format nil "~A~A" *url-jar-pathname-base* ,path))))
    216 
    217 ;;; wrapped in PROGN for easy disabling without a network connection
    218 (progn
    219   (deftest jar-pathname.load.http.1
    220       (load-url-relative "foo")
    221     t)
    222 
    223   (deftest jar-pathname.load.http.2
    224       (load-url-relative "bar")
    225     t)
    226 
    227   (deftest jar-pathname.load.http.3
    228       (load-url-relative "bar.abcl")
    229     t)
    230 
    231   (deftest jar-pathname.load.http.4
    232       (load-url-relative "eek")
    233     t)
    234 
    235   (deftest jar-pathname.load.http.5
    236       (load-url-relative "eek.lisp")
    237     t)
    238 
    239   (deftest jar-pathname.load.http.6
    240       (load-url-relative "a/b/foo")
    241     t)
    242 
    243   (deftest jar-pathname.load.http.7
    244       (load-url-relative "a/b/bar")
    245     t)
    246 
    247   (deftest jar-pathname.load.http.8
    248       (load-url-relative "a/b/bar.abcl")
    249     t)
    250 
    251   (deftest jar-pathname.load.http.9
    252       (load-url-relative "a/b/eek")
    253     t)
    254 
    255   (deftest jar-pathname.load.http.10
    256       (load-url-relative "a/b/eek.lisp")
    257     t))
     211  "jar:https://abcl.org/releases/1.7.1/abcl-bin-1.7.1.zip!/")
     212
     213(deftest jar-pathname.url.https.1
     214    (probe-file *url-jar-pathname-base*)
     215  *url-jar-pathname-base*)
    258216
    259217(deftest jar-pathname.probe-file.1

  • trunk/abcl/test/src/org/armedbear/lisp/ZipTest.java

    r15395 r15397  
    1818public class ZipTest
    1919{
    20   String zipFile;
     20  // FIXME These need to be created as part of executing the tests
     21  String zipFile = "/Users/evenson/work/abcl/dist/abcl-contrib.jar";
    2122  PathnameJar zip;
     23  String nestedJarFile = "/var/tmp/cl-ppcre-2.1.1.jar";
     24  PathnameJar nestedJar;
    2225
    2326  @Before
    2427  public void setup() {
    25     zipFile = "/Users/evenson/work/abcl/dist/abcl-contrib.jar";
    2628    zip = (PathnameJar) PathnameJar.createFromFile(zipFile);
     29    nestedJar = (PathnameJar) PathnameJar.createFromFile(nestedJarFile);
    2730  }
    2831
    … …  
    3134    ZipCache.Archive archive1 = ZipCache.getArchive(zip);
    3235    assertTrue("Get ZipArchive from pathname",
    33                archive1.file != null);
     36               archive1 instanceof ZipCache.ArchiveFile
     37               && ((ZipCache.ArchiveFile)archive1).file != null);
    3438    PathnameJar zip2
    3539      = (PathnameJar) PathnameJar.createFromFile(zipFile);
    3640    ZipCache.Archive archive2 = ZipCache.getArchive(zip2);
    3741    assertTrue("Get cached ZipArchive from pathname",
    38                archive2.file != null);
     42               archive2 instanceof ZipCache.ArchiveFile
     43               && ((ZipCache.ArchiveFile)archive2).file != null);
    3944    assertTrue("Cached ZipArchive refers to same entry",
    4045               archive2.equals(archive1));
    … …  
    5661  }
    5762
    58    
     63
     64  @Test
     65  public void nestedJar() {
     66    String nestedNamestring = "jar:jar:file:/var/tmp/cl-ppcre-2.1.1.jar!/cl-ppcre/packages.abcl!/__loader__._";
     67    Pathname nested = (Pathname)PathnameJar.create(nestedNamestring);
     68  }
     69 
    5970 
    6071
Note: See TracChangeset for help on using the changeset viewer.