maker segment and index policy manager

maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java

+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+//
+// @Author Lion <chenxin619315@gmail.com>
+// @Date   2022/07/14
+
+package org.lionsoul.ip2region.xdb;
+
+public class IndexPolicy {
+    public static final int Vector = 1;
+    public static final int BTree = 2;
+
+    // parser the index policy from string
+    public static int parse(String policy) throws Exception {
+        String v = policy.toLowerCase();
+        if ("vector".equals(v)) {
+            return Vector;
+        } else if ("btree".equals(v)) {
+            return BTree;
+        } else {
+            throw new Exception("unknown index policy `"+policy+"`");
+        }
+    }
+}
\ No newline at end of file

maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java

+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+//
+// @Author Lion <chenxin619315@gmail.com>
+// @Date   2022/07/14
+
+package org.lionsoul.ip2region.xdb;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+// simple log implementation
+public class Log {
+
+    /* Log level constants define */
+    public static final int DEBUG = 0;
+    public static final int INFO = 1;
+    public static final int WARN = 2;
+    public static final int ERROR = 3;
+
+    // level name
+    public static final String[] level_string = new String[] {
+        "DEBUG",
+        "INFO",
+        "WARN",
+        "ERROR"
+    };
+
+    public final Class<?> baseClass;
+    private static int level;
+
+    public Log(Class<?> baseClass) {
+        this.baseClass = baseClass;
+    }
+
+    public static Log getLogger(Class<?> baseClass) {
+        return new Log(baseClass);
+    }
+
+    public String format(int level, String format, Object... args) {
+        // append the datetime
+        final StringBuilder sb = new StringBuilder();
+        final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level]));
+
+        // append the class name
+        sb.append(baseClass.getName()).append(' ');
+        sb.append(String.format(format, args));
+        return sb.toString();
+    }
+
+    public void printf(int level, String format, Object... args) {
+        if (level < DEBUG || level > ERROR) {
+            throw new IndexOutOfBoundsException("invalid level index " + level);
+        }
+
+        // level filter
+        if (level < Log.level) {
+            return;
+        }
+
+        System.out.println(format(level, format, args));
+        System.out.flush();
+    }
+
+    public String getDebugf(String format, Object... args) {
+        return format(DEBUG, format, args);
+    }
+
+    public void debugf(String format, Object... args) {
+        printf(DEBUG, format, args);
+    }
+
+    public String getInfof(String format, Object... args) {
+        return format(INFO, format, args);
+    }
+
+    public void infof(String format, Object... args) {
+        printf(INFO, format, args);
+    }
+
+    public String getWarnf(String format, Object... args) {
+        return format(WARN, format, args);
+    }
+
+    public void warnf(String format, Object... args) {
+        printf(WARN, format, args);
+    }
+
+    public String getErrorf(String format, Object... args) {
+        return format(ERROR, format, args);
+    }
+
+    public void errorf(String format, Object... args) {
+        printf(ERROR, format, args);
+    }
+
+    public static void setLevel(int level) {
+        Log.level = level;
+    }
+
+    public static void setLevel(String level) {
+        String v = level.toLowerCase();
+        if ("debug".equals(v)) {
+            Log.level = DEBUG;
+        } else if ("info".equals(v)) {
+            Log.level = INFO;
+        } else if ("warn".equals(v)) {
+            Log.level = WARN;
+        } else if ("error".equals(v)) {
+            Log.level = ERROR;
+        }
+    }
+
+}
\ No newline at end of file

maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java

@@ -51,6 +51,13 @@
 
 package org.lionsoul.ip2region.xdb;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.HashMap;
+import java.util.Map;
+
 public class Maker {
     // constants define
     public static final int VersionNo = 2;
@@ -60,7 +67,74 @@ public class Maker {
     public static final int VectorIndexSize = 8;
     public static final int SegmentIndexSize = 14;
 
-    public Maker() {
+    private static final Log log = Log.getLogger(Maker.class);
+
+    // source text file handle
+    private final File srcFile;
+
+    // destination binary file handle
+    private final RandomAccessFile dstHandle;
+
+    // index policy
+    private final int indexPolicy;
+
+    // region pool
+    private final Map<String, Long> regionPool;
+
+    // vector index raw bytes
+    private byte[] vectorIndex;
+
+    public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
+        this.srcFile = new File(srcFile);
+        if (!this.srcFile.exists()) {
+            throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
+        }
+
+        this.dstHandle = new RandomAccessFile(dstFile, "r");
+        this.indexPolicy = policy;
+        this.regionPool = new HashMap<String, Long>();
+    }
+
+    // init the header of the target xdb binary file
+    private void initHeader() throws IOException {
+        log.infof("try to init the db header ... ");
+        dstHandle.seek(0);
+
+        // make and write the header space
+        final byte[] header = new byte[HeaderInfoLength];
+
+        // encode the data
+        Util.write(header, 0, VersionNo, 2);
+        Util.write(header, 2, indexPolicy, 2);
+        Util.write(header, 4, System.currentTimeMillis() / 1000, 4);
+        Util.write(header, 8, 0, 4);    // start index ptr
+        Util.write(header, 12, 0, 4);   // end index ptr
+
+        dstHandle.write(header);
+    }
+
+    // load all the segments
+    private void loadSegments() {
+
+    }
+
+    // init the maker
+    public void init() throws IOException {
+        // init the db header
+        initHeader();
+
+        // load all the segments
+        loadSegments();
+    }
+
+    // start to make the binary file
+    public void make() {
+
+    }
+
+    // end the make, do the resource clean up
+    public void end() throws IOException {
+        this.dstHandle.close();
     }
 
 }

maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java

+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+//
+// @Author Lion <chenxin619315@gmail.com>
+// @Date   2022/07/14
+
+package org.lionsoul.ip2region.xdb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Segment {
+    public final long startIP;
+    public final long endIP;
+    public final String region;
+
+    // parser the Segment from an input string
+    public static Segment parse(String input) throws Exception {
+        String[] ps = input.split("\\|", 3);
+        if (ps.length != 3) {
+            throw new Exception("invalid ip segment `"+input+"`");
+        }
+
+        long sip = Util.checkIP(ps[0]);
+        long eip = Util.checkIP(ps[1]);
+        if (sip > eip) {
+            throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`");
+        }
+
+        return new Segment(sip, eip, ps[2]);
+    }
+
+    public Segment(long startIP, long endIP, String region) {
+        this.startIP = startIP;
+        this.endIP = endIP;
+        this.region = region;
+    }
+
+    // split the current segment for vector index
+    public List<Segment> split() {
+        long sByte1 = (int) ((startIP >> 24) & 0xFF);
+        long eByte1 = (int) ((endIP >> 24) & 0xFF);
+        long nSip = startIP;
+        final List<Segment> tList =  new ArrayList<Segment>();
+        for (long i = sByte1; i <= eByte1; i++) {
+            long sip = (i << 24) | (nSip & 0xFFFFFF);
+            long eip = (i << 24) | 0xFFFFFF;
+            if (eip < endIP) {
+                nSip = (i + 1) << 24;
+            } else {
+                eip = endIP;
+            }
+
+            // append the new segment
+            tList.add(new Segment(sip, eip, null));
+        }
+
+        // 2, split the segments with the second byte
+        final List<Segment> segList = new ArrayList<Segment>();
+        for (Segment seg : tList) {
+            long base = seg.startIP & 0xFF000000;
+            long tSip = seg.startIP;
+            long sb2 = (seg.startIP >> 16) & 0xFF;
+            long eb2 = (seg.endIP >> 16) & 0xFF;
+            for (long i = sb2; i <= eb2; i++) {
+                long sip = base | (i << 16) | (tSip & 0xFFFF);
+                long eip = base | (i << 16) | 0xFFFF;
+                if (eip < seg.endIP) {
+                    tSip = 0;
+                } else {
+                    eip = seg.endIP;
+                }
+
+                segList.add(new Segment(sip, eip, seg.region));
+            }
+        }
+
+        return segList;
+    }
+
+    @Override public String toString() {
+        return Util.long2ip(startIP) + "|" + Util.long2ip(endIP) + "|" + region;
+    }
+}

maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java

+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+//
+// @Author Lion <chenxin619315@gmail.com>
+// @Date   2022/07/14
+
+package org.lionsoul.ip2region.xdb;
+
+public class Util
+{
+    // write specified bytes into a byte array start from offset
+    public static void write( byte[] b, int offset, long v, int bytes) {
+        for ( int i = 0; i < bytes; i++ ) {
+            b[offset++] = (byte)((v >>> (8 * i)) & 0xFF);
+        }
+    }
+
+    // write a int to a byte array
+    public static void writeIntLong(byte[] b, int offset, long v) {
+        b[offset++] = (byte)((v      ) & 0xFF);
+        b[offset++] = (byte)((v >>  8) & 0xFF);
+        b[offset++] = (byte)((v >> 16) & 0xFF);
+        b[offset  ] = (byte)((v >> 24) & 0xFF);
+    }
+
+    // get an int from a byte array start from the specified offset
+    public static long getIntLong(byte[] b, int offset) {
+        return (
+            ((b[offset++] & 0x000000FFL)) |
+            ((b[offset++] <<  8) & 0x0000FF00L) |
+            ((b[offset++] << 16) & 0x00FF0000L) |
+            ((b[offset  ] << 24) & 0xFF000000L)
+        );
+    }
+
+    public static int getInt2(byte[] b, int offset) {
+        return (
+            (b[offset++] & 0x000000FF) |
+            (b[offset  ] & 0x0000FF00)
+        );
+    }
+
+    /* long int to ip string */
+    public static String long2ip( long ip ) {
+        return String.valueOf((ip >> 24) & 0xFF) + '.' +
+                ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
+    }
+
+    public static final byte[] shiftIndex = {24, 16, 8, 0};
+
+    /* check the specified ip address */
+    public static long checkIP(String ip) throws Exception {
+        String[] ps = ip.split("\\.");
+        if (ps.length != 4) {
+            throw new Exception("invalid ip address `" + ip + "`");
+        }
+
+        long ipDst = 0;
+        for (int i = 0; i < ps.length; i++) {
+            int val = Integer.parseInt(ps[i]);
+            if (val > 255) {
+                throw new Exception("ip part `"+ps[i]+"` should be less then 256");
+            }
+
+            ipDst |= ((long) val << shiftIndex[i]);
+        }
+
+        return ipDst & 0xFFFFFFFFL;
+    }
+
+}