From 71404d8c342be1dceade048e9687158cfb5add55 Mon Sep 17 00:00:00 2001 From: lion Date: Thu, 14 Jul 2022 16:38:26 +0800 Subject: [PATCH] maker segment and index policy manager --- .../lionsoul/ip2region/xdb/IndexPolicy.java | 25 ++++ .../java/org/lionsoul/ip2region/xdb/Log.java | 116 ++++++++++++++++++ .../org/lionsoul/ip2region/xdb/Maker.java | 76 +++++++++++- .../org/lionsoul/ip2region/xdb/Segment.java | 85 +++++++++++++ .../java/org/lionsoul/ip2region/xdb/Util.java | 72 +++++++++++ 5 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java create mode 100644 maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java new file mode 100644 index 0000000..64ddc11 --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java @@ -0,0 +1,25 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +public class IndexPolicy { + public static final int Vector = 1; + public static final int BTree = 2; + + // parser the index policy from string + public static int parse(String policy) throws Exception { + String v = policy.toLowerCase(); + if ("vector".equals(v)) { + return Vector; + } else if ("btree".equals(v)) { + return BTree; + } else { + throw new Exception("unknown index policy `"+policy+"`"); + } + } +} \ No newline at end of file diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java new file mode 100644 index 0000000..d3a3856 --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java @@ -0,0 +1,116 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +import java.text.SimpleDateFormat; +import java.util.Date; + +// simple log implementation +public class Log { + + /* Log level constants define */ + public static final int DEBUG = 0; + public static final int INFO = 1; + public static final int WARN = 2; + public static final int ERROR = 3; + + // level name + public static final String[] level_string = new String[] { + "DEBUG", + "INFO", + "WARN", + "ERROR" + }; + + public final Class baseClass; + private static int level; + + public Log(Class baseClass) { + this.baseClass = baseClass; + } + + public static Log getLogger(Class baseClass) { + return new Log(baseClass); + } + + public String format(int level, String format, Object... args) { + // append the datetime + final StringBuilder sb = new StringBuilder(); + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level])); + + // append the class name + sb.append(baseClass.getName()).append(' '); + sb.append(String.format(format, args)); + return sb.toString(); + } + + public void printf(int level, String format, Object... args) { + if (level < DEBUG || level > ERROR) { + throw new IndexOutOfBoundsException("invalid level index " + level); + } + + // level filter + if (level < Log.level) { + return; + } + + System.out.println(format(level, format, args)); + System.out.flush(); + } + + public String getDebugf(String format, Object... args) { + return format(DEBUG, format, args); + } + + public void debugf(String format, Object... args) { + printf(DEBUG, format, args); + } + + public String getInfof(String format, Object... args) { + return format(INFO, format, args); + } + + public void infof(String format, Object... args) { + printf(INFO, format, args); + } + + public String getWarnf(String format, Object... args) { + return format(WARN, format, args); + } + + public void warnf(String format, Object... args) { + printf(WARN, format, args); + } + + public String getErrorf(String format, Object... args) { + return format(ERROR, format, args); + } + + public void errorf(String format, Object... args) { + printf(ERROR, format, args); + } + + public static void setLevel(int level) { + Log.level = level; + } + + public static void setLevel(String level) { + String v = level.toLowerCase(); + if ("debug".equals(v)) { + Log.level = DEBUG; + } else if ("info".equals(v)) { + Log.level = INFO; + } else if ("warn".equals(v)) { + Log.level = WARN; + } else if ("error".equals(v)) { + Log.level = ERROR; + } + } + +} \ No newline at end of file diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java index 23f0d47..d86f427 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java @@ -51,6 +51,13 @@ package org.lionsoul.ip2region.xdb; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.HashMap; +import java.util.Map; + public class Maker { // constants define public static final int VersionNo = 2; @@ -60,7 +67,74 @@ public class Maker { public static final int VectorIndexSize = 8; public static final int SegmentIndexSize = 14; - public Maker() { + private static final Log log = Log.getLogger(Maker.class); + + // source text file handle + private final File srcFile; + + // destination binary file handle + private final RandomAccessFile dstHandle; + + // index policy + private final int indexPolicy; + + // region pool + private final Map regionPool; + + // vector index raw bytes + private byte[] vectorIndex; + + public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException { + this.srcFile = new File(srcFile); + if (!this.srcFile.exists()) { + throw new FileNotFoundException("source text file `" +srcFile+ "` not found"); + } + + this.dstHandle = new RandomAccessFile(dstFile, "r"); + this.indexPolicy = policy; + this.regionPool = new HashMap(); + } + + // init the header of the target xdb binary file + private void initHeader() throws IOException { + log.infof("try to init the db header ... "); + dstHandle.seek(0); + + // make and write the header space + final byte[] header = new byte[HeaderInfoLength]; + + // encode the data + Util.write(header, 0, VersionNo, 2); + Util.write(header, 2, indexPolicy, 2); + Util.write(header, 4, System.currentTimeMillis() / 1000, 4); + Util.write(header, 8, 0, 4); // start index ptr + Util.write(header, 12, 0, 4); // end index ptr + + dstHandle.write(header); + } + + // load all the segments + private void loadSegments() { + + } + + // init the maker + public void init() throws IOException { + // init the db header + initHeader(); + + // load all the segments + loadSegments(); + } + + // start to make the binary file + public void make() { + + } + + // end the make, do the resource clean up + public void end() throws IOException { + this.dstHandle.close(); } } diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java new file mode 100644 index 0000000..43fdf50 --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java @@ -0,0 +1,85 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +import java.util.ArrayList; +import java.util.List; + +public class Segment { + public final long startIP; + public final long endIP; + public final String region; + + // parser the Segment from an input string + public static Segment parse(String input) throws Exception { + String[] ps = input.split("\\|", 3); + if (ps.length != 3) { + throw new Exception("invalid ip segment `"+input+"`"); + } + + long sip = Util.checkIP(ps[0]); + long eip = Util.checkIP(ps[1]); + if (sip > eip) { + throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`"); + } + + return new Segment(sip, eip, ps[2]); + } + + public Segment(long startIP, long endIP, String region) { + this.startIP = startIP; + this.endIP = endIP; + this.region = region; + } + + // split the current segment for vector index + public List split() { + long sByte1 = (int) ((startIP >> 24) & 0xFF); + long eByte1 = (int) ((endIP >> 24) & 0xFF); + long nSip = startIP; + final List tList = new ArrayList(); + for (long i = sByte1; i <= eByte1; i++) { + long sip = (i << 24) | (nSip & 0xFFFFFF); + long eip = (i << 24) | 0xFFFFFF; + if (eip < endIP) { + nSip = (i + 1) << 24; + } else { + eip = endIP; + } + + // append the new segment + tList.add(new Segment(sip, eip, null)); + } + + // 2, split the segments with the second byte + final List segList = new ArrayList(); + for (Segment seg : tList) { + long base = seg.startIP & 0xFF000000; + long tSip = seg.startIP; + long sb2 = (seg.startIP >> 16) & 0xFF; + long eb2 = (seg.endIP >> 16) & 0xFF; + for (long i = sb2; i <= eb2; i++) { + long sip = base | (i << 16) | (tSip & 0xFFFF); + long eip = base | (i << 16) | 0xFFFF; + if (eip < seg.endIP) { + tSip = 0; + } else { + eip = seg.endIP; + } + + segList.add(new Segment(sip, eip, seg.region)); + } + } + + return segList; + } + + @Override public String toString() { + return Util.long2ip(startIP) + "|" + Util.long2ip(endIP) + "|" + region; + } +} diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java new file mode 100644 index 0000000..33d6f5a --- /dev/null +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java @@ -0,0 +1,72 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// +// @Author Lion +// @Date 2022/07/14 + +package org.lionsoul.ip2region.xdb; + +public class Util +{ + // write specified bytes into a byte array start from offset + public static void write( byte[] b, int offset, long v, int bytes) { + for ( int i = 0; i < bytes; i++ ) { + b[offset++] = (byte)((v >>> (8 * i)) & 0xFF); + } + } + + // write a int to a byte array + public static void writeIntLong(byte[] b, int offset, long v) { + b[offset++] = (byte)((v ) & 0xFF); + b[offset++] = (byte)((v >> 8) & 0xFF); + b[offset++] = (byte)((v >> 16) & 0xFF); + b[offset ] = (byte)((v >> 24) & 0xFF); + } + + // get an int from a byte array start from the specified offset + public static long getIntLong(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset ] << 24) & 0xFF000000L) + ); + } + + public static int getInt2(byte[] b, int offset) { + return ( + (b[offset++] & 0x000000FF) | + (b[offset ] & 0x0000FF00) + ); + } + + /* long int to ip string */ + public static String long2ip( long ip ) { + return String.valueOf((ip >> 24) & 0xFF) + '.' + + ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); + } + + public static final byte[] shiftIndex = {24, 16, 8, 0}; + + /* check the specified ip address */ + public static long checkIP(String ip) throws Exception { + String[] ps = ip.split("\\."); + if (ps.length != 4) { + throw new Exception("invalid ip address `" + ip + "`"); + } + + long ipDst = 0; + for (int i = 0; i < ps.length; i++) { + int val = Integer.parseInt(ps[i]); + if (val > 255) { + throw new Exception("ip part `"+ps[i]+"` should be less then 256"); + } + + ipDst |= ((long) val << shiftIndex[i]); + } + + return ipDst & 0xFFFFFFFFL; + } + +} -- GitLab