Maven cache & resolve

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/driver/DriverLibraryMavenArtifact.java

@@ -133,25 +133,12 @@ public class DriverLibraryMavenArtifact extends DriverLibraryAbstract
             List<MavenArtifactDependency> artifactDeps = metaData.getDependencies(monitor);
             if (!CommonUtils.isEmpty(artifactDeps)) {
                 for (MavenArtifactDependency artifactDep : artifactDeps) {
-                    if (artifactDep.isOptional()) {
-                        continue;
-                    }
-                    switch (artifactDep.getScope()) {
-                        case COMPILE:
-                        case RUNTIME:
-                        {
-                            MavenLocalVersion depLocalVersion = artifactDep.resolveDependency(monitor);
-                            if (depLocalVersion != null) {
-                                dependencies.add(
-                                    new DriverLibraryMavenDependency(
-                                        this.getDriver(),
-                                        depLocalVersion));
-                            }
-                            break;
-                        }
-                        default:
-                            // We don't need it
-                            break;
+                    MavenLocalVersion depLocalVersion = artifactDep.resolveDependency(monitor);
+                    if (depLocalVersion != null) {
+                        dependencies.add(
+                            new DriverLibraryMavenDependency(
+                                this.getDriver(),
+                                depLocalVersion));
                     }
                 }
             }

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/driver/DriverLibraryMavenDependency.java

@@ -34,7 +34,7 @@ public class DriverLibraryMavenDependency extends DriverLibraryMavenArtifact
     private MavenLocalVersion localVersion;
 
     public DriverLibraryMavenDependency(DriverDescriptor driverDescriptor, MavenLocalVersion localVersion) {
-        super(driverDescriptor, FileType.jar, PATH_PREFIX + localVersion.getArtifact().toString());
+        super(driverDescriptor, FileType.jar, PATH_PREFIX + localVersion.toString());
         this.localVersion = localVersion;
     }
 
@@ -64,7 +64,11 @@ public class DriverLibraryMavenDependency extends DriverLibraryMavenArtifact
 
     @NotNull
     public String getDisplayName() {
-        return localVersion.getArtifact().getGroupId() + ":" + localVersion.getArtifact().getArtifactId() + ":" + localVersion.getVersion();
+        return localVersion.getArtifact().getGroupId() + ":" + localVersion.getArtifact().getArtifactId();
+    }
+
+    public String getVersion() {
+        return localVersion.getVersion();
     }
 
     protected MavenLocalVersion resolveLocalVersion(DBRProgressMonitor monitor, boolean forceUpdate) throws IOException {

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/maven/MavenArtifact.java

@@ -259,7 +259,10 @@ public class MavenArtifact
             }
         }
 
-        MavenLocalVersion localVersion = getActiveLocalVersion();
+        MavenLocalVersion localVersion = getLocalVersion(versionInfo);
+        if (localVersion == null && lookupVersion) {
+            localVersion = getActiveLocalVersion();
+        }
         if (localVersion == null) {
             localVersion = makeLocalVersion(monitor, versionInfo, true);
         }

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/maven/MavenArtifactVersion.java

@@ -19,7 +19,7 @@ package org.jkiss.dbeaver.registry.maven;
 
 import org.jkiss.dbeaver.Log;
 import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
-import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor;
+import org.jkiss.dbeaver.registry.driver.DriverLibraryMavenDependency;
 import org.jkiss.dbeaver.runtime.RuntimeUtils;
 import org.jkiss.dbeaver.utils.GeneralUtils;
 import org.jkiss.utils.CommonUtils;
@@ -48,7 +48,6 @@ public class MavenArtifactVersion {
     private String description;
     private String url;
     private MavenLocalVersion parent;
-    private MavenArtifactReference parentReference;
     private Map<String, String> properties = new LinkedHashMap<>();
     private List<MavenArtifactLicense> licenses = new ArrayList<>();
     private List<MavenArtifactDependency> dependencies;
@@ -66,7 +65,7 @@ public class MavenArtifactVersion {
                 } else if (name.equals(PROP_PROJECT_ARTIFACT_ID)) {
                     value = localVersion.getArtifact().getArtifactId();
                 } else if (parent != null) {
-                    return parent.getMetaData(VoidProgressMonitor.INSTANCE).variableResolver.get(name);
+                    return parent.getMetaData().variableResolver.get(name);
                 }
             }
             return value;
@@ -78,7 +77,8 @@ public class MavenArtifactVersion {
         loadPOM(monitor);
     }
 
-    MavenArtifactVersion(String name, String version) {
+    MavenArtifactVersion(MavenLocalVersion localVersion, String name, String version) {
+        this.localVersion = localVersion;
         this.name = name;
         this.version = version;
     }
@@ -103,8 +103,19 @@ public class MavenArtifactVersion {
         return url;
     }
 
-    public MavenArtifactReference getParentReference() {
-        return parentReference;
+    public MavenLocalVersion getParent() {
+        return parent;
+    }
+
+    void setParent(MavenLocalVersion parent) {
+        this.parent = parent;
+    }
+
+    void addDependency(MavenArtifactDependency dependency) {
+        if (dependencies == null) {
+            dependencies = new ArrayList<>();
+        }
+        dependencies.add(dependency);
     }
 
     public Map<String, String> getProperties() {
@@ -164,7 +175,7 @@ public class MavenArtifactVersion {
                 if (parentGroupId == null || parentArtifactId == null || parentVersion == null) {
                     log.error("Broken parent reference: " + parentGroupId + ":" + parentArtifactId + ":" + parentVersion);
                 } else {
-                    parentReference = new MavenArtifactReference(
+                    MavenArtifactReference parentReference = new MavenArtifactReference(
                         parentGroupId,
                         parentArtifactId,
                         parentVersion
@@ -208,14 +219,14 @@ public class MavenArtifactVersion {
             // Dependencies
             Element dmElement = XMLUtils.getChildElement(root, "dependencyManagement");
             if (dmElement != null) {
-                dependencyManagement = parseDependencies(monitor, dmElement);
+                dependencyManagement = parseDependencies(monitor, dmElement, true);
             }
-            dependencies = parseDependencies(monitor, root);
+            dependencies = parseDependencies(monitor, root, false);
         }
         monitor.worked(1);
     }
 
-    private List<MavenArtifactDependency> parseDependencies(DBRProgressMonitor monitor, Element element) {
+    private List<MavenArtifactDependency> parseDependencies(DBRProgressMonitor monitor, Element element, boolean all) {
         List<MavenArtifactDependency> result = new ArrayList<>();
         Element dependenciesElement = XMLUtils.getChildElement(element, "dependencies");
         if (dependenciesElement != null) {
@@ -239,13 +250,18 @@ public class MavenArtifactVersion {
                 if (!CommonUtils.isEmpty(scopeName)) {
                     scope = MavenArtifactDependency.Scope.valueOf(scopeName.toUpperCase(Locale.ENGLISH));
                 }
-                result.add(new MavenArtifactDependency(
-                    evaluateString(groupId),
-                    evaluateString(artifactId),
-                    evaluateString(version),
-                    scope,
-                    CommonUtils.getBoolean(XMLUtils.getChildElementBody(dep, "optional"), false)
-                ));
+                boolean optional = CommonUtils.getBoolean(XMLUtils.getChildElementBody(dep, "optional"), false);
+
+                // TODO: maybe we should include some of them
+                if (all || (!optional && (scope == MavenArtifactDependency.Scope.COMPILE || scope == MavenArtifactDependency.Scope.RUNTIME))) {
+                    result.add(new MavenArtifactDependency(
+                        evaluateString(groupId),
+                        evaluateString(artifactId),
+                        evaluateString(version),
+                        scope,
+                        optional
+                    ));
+                }
             }
         }
         return result;

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/maven/MavenLocalVersion.java

@@ -88,19 +88,27 @@ public class MavenLocalVersion
                 metaData = new MavenArtifactVersion(monitor, this);
             } catch (IOException e) {
                 log.warn("Error fetching POM file", e);
-                metaData = new MavenArtifactVersion(artifact.getArtifactId(), version);
+                metaData = new MavenArtifactVersion(this, artifact.getArtifactId(), version);
             }
         }
         return metaData;
     }
 
+    public String getPath() {
+        return artifact.toString() + ":" + version;
+    }
+
     MavenArtifactVersion getMetaData() {
         return metaData;
     }
 
+    void setMetaData(MavenArtifactVersion metaData) {
+        this.metaData = metaData;
+    }
+
     @Override
     public String toString() {
-        return artifact.toString() + ":" + version;
+        return getPath();
     }
 
 }

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/maven/MavenRegistry.java

@@ -44,18 +44,29 @@ public class MavenRegistry
     {
         if (instance == null) {
             instance = new MavenRegistry();
-            instance.loadCustomRepositories();
+            instance.init();
         }
         return instance;
     }
 
     private final List<MavenRepository> repositories = new ArrayList<MavenRepository>();
-    private final MavenRepository localRepository;
+    private MavenRepository localRepository;
     // Cache for not found artifact ids. Avoid multiple remote metadata reading
     private final Set<String> notFoundArtifacts = new HashSet<String>();
 
     private MavenRegistry()
     {
+    }
+
+    private void init() {
+        loadStandardRepositories();
+        loadCustomRepositories();
+        loadCache();
+        // Start config saver
+        new ConfigSaver().schedule(ConfigSaver.SAVE_PERIOD);
+    }
+
+    private void loadStandardRepositories() {
         // Load repositories info
         {
             IConfigurationElement[] extElements = Platform.getExtensionRegistry().getConfigurationElementsFor(MavenRepository.EXTENSION_ID);
@@ -76,9 +87,13 @@ public class MavenRegistry
             MAVEN_LOCAL_REPO_NAME,
             localRepoURL,
             true);
+    }
 
-        // Start config saver
-        new ConfigSaver().schedule(ConfigSaver.SAVE_PERIOD);
+    private void loadCache() {
+        localRepository.loadCache();
+        for (MavenRepository repository : repositories) {
+            repository.loadCache();
+        }
     }
 
     public void loadCustomRepositories() {

plugins/org.jkiss.dbeaver.core/src/org/jkiss/dbeaver/registry/maven/MavenRepository.java

@@ -33,6 +33,9 @@ import org.jkiss.utils.xml.XMLException;
 import org.xml.sax.Attributes;
 
 import java.io.*;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.*;
 
 /**
@@ -46,10 +49,10 @@ public class MavenRepository
 
     public static final String METADATA_CACHE_FILE = "metadata-cache.xml";
 
-    private static final String TAG_CACHE = "cache";
-    private static final String TAG_ARTIFACT = "artifact";
-    private static final String TAG_VERSION = "version";
-    private static final String TAG_DEPENDENCY = "dependency";
+    public static final String TAG_CACHE = "cache";
+    public static final String TAG_ARTIFACT = "artifact";
+    public static final String TAG_VERSION = "version";
+    public static final String TAG_DEPENDENCY = "dependency";
 
     public static final String ATTR_NAME = "name";
     public static final String ATTR_URL = "url";
@@ -59,9 +62,12 @@ public class MavenRepository
     public static final String ATTR_VERSION = "version";
     public static final String ATTR_PATH = "path";
     public static final String ATTR_SCOPE = "scope";
-    private static final String ATTR_PARENT = "parent";
+    public static final String ATTR_OPTIONAL = "optional";
+    public static final String ATTR_PARENT = "parent";
     public static final String ATTR_UPDATE_TIME = "updateTime";
 
+    private static final DateFormat UPDATE_TIME_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss");
+
     private String id;
     private String name;
     private String url;
@@ -88,7 +94,6 @@ public class MavenRepository
         if (!url.endsWith("/")) url += "/";
         this.url = url;
         this.local = local;
-        loadCache();
     }
 
     public void flushCache() {
@@ -158,17 +163,52 @@ public class MavenRepository
         return homeFolder;
     }
 
-    synchronized private void loadCache() {
+    private static class DependencyResolveInfo {
+        String path;
+        MavenArtifactDependency.Scope scope;
+        boolean optional;
+
+        DependencyResolveInfo(String path, MavenArtifactDependency.Scope scope, boolean optional) {
+            this.path = path;
+            this.scope = scope;
+            this.optional = optional;
+        }
+
+        @Override
+        public String toString() {
+            return path;
+        }
+    }
+
+    private static class VersionResolveInfo {
+        MavenLocalVersion localVersion;
+        String parentPath;
+        List<DependencyResolveInfo> dependencies = new ArrayList<>();
+
+        public VersionResolveInfo(MavenLocalVersion localVersion, String parentPath) {
+            this.localVersion = localVersion;
+            this.parentPath = parentPath;
+        }
+
+        @Override
+        public String toString() {
+            return localVersion.toString();
+        }
+    }
+
+    synchronized void loadCache() {
         File cacheFile = new File(getLocalCacheDir(), METADATA_CACHE_FILE);
         if (!cacheFile.exists()) {
             return;
         }
+        final List<VersionResolveInfo> lateResolutions = new ArrayList<>();
         try {
             InputStream mdStream = new FileInputStream(cacheFile);
             try {
                 SAXReader reader = new SAXReader(mdStream);
                 reader.parse(new SAXListener() {
-                    public MavenArtifact lastArtifact;
+                    MavenArtifact lastArtifact;
+                    VersionResolveInfo lastVersionResolveInfo;
                     @Override
                     public void saxStartElement(SAXReader reader, String namespaceURI, String localName, Attributes atts) throws XMLException {
                         if (TAG_ARTIFACT.equals(localName)) {
@@ -179,11 +219,38 @@ public class MavenRepository
                             lastArtifact.setActiveVersion(atts.getValue(ATTR_ACTIVE_VERSION));
                             cachedArtifacts.add(lastArtifact);
                         } else if (TAG_VERSION.equals(localName) && lastArtifact != null) {
+                            Date updateTime = new Date();
+                            try {
+                                updateTime = UPDATE_TIME_FORMAT.parse(atts.getValue(ATTR_UPDATE_TIME));
+                            } catch (ParseException e) {
+                                // ignore
+                            }
+                            String versionNumber = atts.getValue(ATTR_VERSION);
                             MavenLocalVersion version = new MavenLocalVersion(
                                 lastArtifact,
-                                atts.getValue(ATTR_VERSION),
-                                new Date(Long.parseLong(atts.getValue(ATTR_UPDATE_TIME))));
+                                versionNumber,
+                                updateTime);
                             lastArtifact.addLocalVersion(version);
+
+                            MavenArtifactVersion lastVersion = new MavenArtifactVersion(version, lastArtifact.getArtifactId(), versionNumber);
+                            version.setMetaData(lastVersion);
+                            lastVersionResolveInfo = new VersionResolveInfo(version, atts.getValue(ATTR_PARENT));
+                            lateResolutions.add(lastVersionResolveInfo);
+                        } else if (TAG_DEPENDENCY.equals(localName) && lastVersionResolveInfo != null) {
+                            MavenArtifactDependency.Scope scope = MavenArtifactDependency.Scope.COMPILE;
+                            String scopeString = atts.getValue(ATTR_SCOPE);
+                            if (scopeString != null) {
+                                try {
+                                    scope = MavenArtifactDependency.Scope.valueOf(scopeString.toUpperCase(Locale.ENGLISH));
+                                } catch (IllegalArgumentException e) {
+                                    log.debug(e);
+                                }
+                            }
+                            lastVersionResolveInfo.dependencies.add(new DependencyResolveInfo(
+                                atts.getValue(ATTR_PATH),
+                                scope,
+                                CommonUtils.getBoolean(atts.getValue(ATTR_OPTIONAL), false)
+                            ));
                         }
                     }
                     @Override
@@ -194,6 +261,8 @@ public class MavenRepository
                     public void saxEndElement(SAXReader reader, String namespaceURI, String localName) throws XMLException {
                         if (TAG_ARTIFACT.equals(localName)) {
                             lastArtifact = null;
+                        } else if (TAG_VERSION.equals(localName)) {
+                            lastVersionResolveInfo = null;
                         }
                     }
                 });
@@ -205,6 +274,43 @@ public class MavenRepository
         } catch (IOException e) {
             log.warn("IO error while reading cached Maven repository '" + id + "'", e);
         }
+
+        // Perform late resolution
+        for (VersionResolveInfo vri : lateResolutions) {
+            if (vri.parentPath != null) {
+                MavenLocalVersion parentVersion = resolveLocalVersion(vri.parentPath);
+                if (parentVersion != null) {
+                    vri.localVersion.getMetaData().setParent(parentVersion);
+                }
+            }
+            for (DependencyResolveInfo dri : vri.dependencies) {
+                MavenLocalVersion depVersion = resolveLocalVersion(dri.path);
+                if (depVersion != null) {
+                    vri.localVersion.getMetaData().addDependency(
+                        new MavenArtifactDependency(
+                            depVersion.getArtifact().getGroupId(),
+                            depVersion.getArtifact().getArtifactId(),
+                            depVersion.getVersion(),
+                            dri.scope,
+                            dri.optional));
+                }
+            }
+        }
+    }
+
+    private MavenLocalVersion resolveLocalVersion(String path) {
+        MavenArtifactReference parentRef = new MavenArtifactReference(path);
+        MavenArtifact parentArtifact = MavenRegistry.getInstance().findArtifact(parentRef);
+        if (parentArtifact == null) {
+            log.warn("Can't resolve artifact " + parentRef);
+            return null;
+        } else {
+            MavenLocalVersion localVersion = parentArtifact.getLocalVersion(parentRef.getVersion());
+            if (localVersion == null) {
+                log.warn("Can't resolve artifact version " + parentRef);
+            }
+            return localVersion;
+        }
     }
 
     void saveCacheIfNeeded() {
@@ -242,11 +348,11 @@ public class MavenRepository
                             for (MavenLocalVersion version : artifact.getLocalVersions()) {
                                 try (XMLBuilder.Element e2 = xml.startElement(TAG_VERSION)) {
                                     xml.addAttribute(ATTR_VERSION, version.getVersion());
-                                    xml.addAttribute(ATTR_UPDATE_TIME, version.getUpdateTime().getTime());
+                                    xml.addAttribute(ATTR_UPDATE_TIME, UPDATE_TIME_FORMAT.format(version.getUpdateTime()));
 
                                     MavenArtifactVersion metaData = version.getMetaData();
                                     if (metaData != null) {
-                                        MavenArtifactReference parentReference = metaData.getParentReference();
+                                        MavenLocalVersion parentReference = metaData.getParent();
                                         if (parentReference != null) {
                                             xml.addAttribute(ATTR_PARENT, parentReference.getPath());
                                         }
@@ -258,6 +364,9 @@ public class MavenRepository
                                                     if (dependency.getScope() != MavenArtifactDependency.Scope.COMPILE) {
                                                         xml.addAttribute(ATTR_SCOPE, dependency.getScope().name().toLowerCase(Locale.ENGLISH));
                                                     }
+                                                    if (dependency.isOptional()) {
+                                                        xml.addAttribute(ATTR_OPTIONAL, true);
+                                                    }
                                                 }
                                             }
                                         }

plugins/org.jkiss.dbeaver.model/src/org/jkiss/dbeaver/utils/GeneralUtils.java

@@ -252,19 +252,28 @@ public class GeneralUtils {
     }
 
     public static String replaceVariables(String string, IVariableResolver resolver) {
-        Matcher matcher = VAR_PATTERN.matcher(string);
-        int pos = 0;
-        while (matcher.find(pos)) {
-            pos = matcher.end();
-            String varName = matcher.group(2);
-            String varValue = resolver.get(varName);
-            if (varValue != null) {
-                matcher = VAR_PATTERN.matcher(
-                        string = matcher.replaceFirst(varValue));
-                pos = 0;
+        try {
+            Matcher matcher = VAR_PATTERN.matcher(string);
+            int pos = 0;
+            while (matcher.find(pos)) {
+                pos = matcher.end();
+                String varName = matcher.group(2);
+                String varValue = resolver.get(varName);
+                if (varValue != null) {
+                    if (matcher.start() == 0 && matcher.end() == string.length() - 1) {
+                        string = varValue;
+                    } else {
+                        string = string.substring(0, matcher.start()) + varValue + string.substring(matcher.end());
+                    }
+                    matcher = VAR_PATTERN.matcher(string);
+                    pos = 0;
+                }
             }
+            return string;
+        } catch (Exception e) {
+            log.warn("Error matching regex", e);
+            return string;
         }
-        return string;
     }
 
     public static String[] parseCommandLine(String commandLine) {