Skip to content

I
Ip2region

int / Ip2region
上一次同步 11 个月

通知 19
Star 0
Fork 0
I
Ip2region

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 8d03749a 编写于 作者: L lion
浏览文件
操作

add java xdb searcher impl

上级 1b6842a2
隐藏空白更改
内联 并排
Showing with 633 addition and 0 deletion
binding/java/pom.xml 0 → 100644
浏览文件 @ 8d03749a
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.lionsoul</groupId>
<artifactId>ip2region</artifactId>
<version>2.0.1</version>
<packaging>jar</packaging>
<name>ip2region</name>
<url>https://gitee.com/lionsoul/ip2region/</url>
<description>Open source internet address db manager framework and locator</description>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<url>git@gitee.com:lionsoul/ip2region.git</url>
<connection>scm:git:git@gitee.com:lionsoul/ip2region.git</connection>
<developerConnection>scm:git:git@gitee.com:lionsoul/ip2region.git</developerConnection>
</scm>
<distributionManagement>
<snapshotRepository>
<id>oss-parent</id>
<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>oss-parent</id>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
<developers>
<developer>
<id>lionsoul</id>
<name>chenxin</name>
<email>chenxin619315@gmail.com</email>
</developer>
</developers>
<issueManagement>
<url>https://gitee.com/lionsoul/ip2region/issues</url>
<system>Gitee issues</system>
</issueManagement>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.1.2</version>
<executions>
<execution>
<id>attach-sources</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
<configuration>
<failOnError>false</failOnError>
<additionalparam>-Xdoclint:none</additionalparam>
<encoding>UTF-8</encoding>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<additionalparam>${javadoc.opts}</additionalparam>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>1.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.lionsoul.ip2region.SearchTest</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.6</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.8</version>
<extensions>true</extensions>
<configuration>
<serverId>oss-parent</serverId>
<nexusUrl>https://oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java 0 → 100644
浏览文件 @ 8d03749a
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/23
package org.lionsoul.ip2region;
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.concurrent.TimeUnit;
public class SearchTest {
public static void printHelp(String[] args) {
System.out.print("ip2region xdb searcher\n");
System.out.print("java -jar ip2region-{version}.jar [command] [command options]\n");
System.out.print("Command: \n");
System.out.print(" search search input test\n");
System.out.print(" bench search bench test\n");
}
public static Searcher createSearcher(String dbPath, String cachePolicy) throws IOException {
if ("file".equals(cachePolicy)) {
return Searcher.newWithFileOnly(dbPath);
} else if ("vectorIndex".equals(cachePolicy)) {
byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath);
return Searcher.newWithVectorIndex(dbPath, vIndex);
} else if ("content".equals(cachePolicy)) {
byte[] cBuff = Searcher.loadContentFromFile(dbPath);
return Searcher.newWithBuffer(cBuff);
} else {
throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content");
}
}
public static void searchTest(String[] args) throws IOException {
String dbPath = "", cachePolicy = "vectorIndex";
for (final String r : args) {
if (r.length() < 5) {
continue;
}
if (r.indexOf("--") != 0) {
continue;
}
int sIdx = r.indexOf('=');
if (sIdx < 0) {
System.out.printf("missing = for args pair `%s`\n", r);
return;
}
String key = r.substring(2, sIdx);
String val = r.substring(sIdx + 1);
System.out.printf("key=%s, val=%s\n", key, val);
if ("db".equals(key)) {
dbPath = val;
} else if ("cache-policy".equals(key)) {
cachePolicy = val;
} else {
System.out.printf("undefined option `%s`", r);
return;
}
}
if (dbPath.length() < 1) {
System.out.print("java -jar ip2region-{version}.jar search [command options]\n");
System.out.print("options:\n");
System.out.print(" --db string ip2region binary xdb file path\n");
System.out.print(" --cache-policy string cache policy: file/vectorIndex/content\n");
return;
}
Searcher searcher = createSearcher(dbPath, cachePolicy);
final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy);
while ( true ) {
System.out.print("ip2region>> ");
String line = reader.readLine().trim();
if ( line.length() < 2 ) {
continue;
}
if ( line.equalsIgnoreCase("quit") ) {
break;
}
try {
double sTime = System.nanoTime();
String region = searcher.searchByStr(line);
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)));
} catch (Exception e) {
System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount());
}
}
reader.close();
searcher.close();
System.out.println("searcher test program exited, thanks for trying");
}
public static void benchTest(String[] args) {
}
public static void main(String[] args) {
if (args.length < 1) {
printHelp(args);
return;
}
if ("search".equals(args[0])) {
try {
searchTest(args);
} catch (IOException e) {
System.out.printf("failed running search test: %s", e);
}
} else if ("bench".equals(args[0])) {
benchTest(args);
} else {
printHelp(args);
}
}
}
binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java 0 → 100644
浏览文件 @ 8d03749a
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/23
package org.lionsoul.ip2region;
import org.lionsoul.ip2region.xdb.Searcher;
import java.io.IOException;
public class UtilTest {
public static void testIP2Long() {
String ip = "1.2.3.4";
long ipAddr = 0;
try {
ipAddr = Searcher.checkIpAddr(ip);
} catch (Exception e) {
System.out.printf("failed to check ip: %s\n", e);
return;
}
if (ipAddr != 16909060) {
System.out.print("failed ip2long\n");
return;
}
String ip2 = Searcher.long2ip(ipAddr);
if (!ip.equals(ip2)) {
System.out.print("failed long2ip\n");
return;
}
System.out.printf("passed: ip=%s, ipAddr=%d, ip2=%s\n", ip, ipAddr, ip2);
}
public static void main(String[] args) {
System.out.print("testing IP2Long ... \n");
testIP2Long();
}
}
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java 0 → 100644
浏览文件 @ 8d03749a
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/23
package org.lionsoul.ip2region.xdb;
import java.awt.image.SampleModel;
public class Header {
public final int version;
public final int indexPolicy;
public final int createdAt;
public final int startIndexPtr;
public final int endIndexPtr;
public Header(byte[] buff) {
assert buff.length >= 16;
version = Searcher.getInt2(buff, 0);
indexPolicy = Searcher.getInt2(buff, 2);
createdAt = Searcher.getInt(buff, 4);
startIndexPtr = Searcher.getInt(buff, 8);
endIndexPtr = Searcher.getInt(buff, 12);
}
@Override public String toString() {
return "{" +
"Version: " + version + ',' +
"IndexPolicy" + indexPolicy + ',' +
"CreatedAt" + createdAt + ',' +
"StartIndexPtr" + startIndexPtr + ',' +
"EndIndexPtr" + endIndexPtr +
'}';
}
}
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java 0 → 100644
浏览文件 @ 8d03749a
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
package org.lionsoul.ip2region.xdb;
// xdb searcher (Not thread safe implementation)
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/23
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Arrays;
public class Searcher {
// constant defined copied from the xdb maker
public static final int HeaderInfoLength = 256;
public static final int VectorIndexRows = 256;
public static final int VectorIndexCols = 256;
public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14;
// random access file handle for file based search
private final RandomAccessFile handle;
private int ioCount = 0;
// vector index.
// use the byte[] instead of VectorIndex entry array to keep
// the minimal memory allocation.
private final byte[] vectorIndex;
// xdb content buffer, used for in-memory search
private final byte[] contentBuff;
// --- static method to create searchers
public static Searcher newWithFileOnly(String dbPath) throws IOException {
return new Searcher(dbPath, null, null);
}
public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException {
return new Searcher(dbPath, vectorIndex, null);
}
public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
return new Searcher(null, null, cBuff);
}
// ---
public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
if (cBuff != null) {
this.handle = null;
this.vectorIndex = null;
this.contentBuff = cBuff;
} else {
this.handle = new RandomAccessFile(dbFile, "r");
this.vectorIndex = vectorIndex;
this.contentBuff = null;
}
}
public void close() throws IOException {
if (this.handle != null) {
this.handle.close();
}
}
public int getIOCount() {
return ioCount;
}
public String searchByStr(String ip) throws Exception {
long ipAddr = checkIpAddr(ip);
return search(ipAddr);
}
public String search(long ip) throws IOException {
// reset the global counter
this.ioCount = 0;
// locate the segment index block based on the vector index
int sPtr = 0, ePtr = 0;
int il0 = (int) ((ip >> 24) & 0xFF);
int il1 = (int) ((ip >> 16) & 0xFF);
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
// System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
if (vectorIndex != null) {
sPtr = getInt(vectorIndex, idx);
ePtr = getInt(vectorIndex, idx + 4);
} else if (contentBuff != null) {
sPtr = getInt(contentBuff, HeaderInfoLength + idx);
ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4);
} else {
final byte[] buff = new byte[8];
read(HeaderInfoLength + idx, buff);
sPtr = getInt(buff, 0);
ePtr = getInt(buff, 4);
}
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
// binary search the segment index block to get the region info
final byte[] buff = new byte[SegmentIndexSize];
int dataLen = -1, dataPtr = -1;
int l = 0, h = (ePtr - sPtr) / SegmentIndexSize;
while (l <= h) {
int m = (l + h) >> 1;
int p = sPtr + m * SegmentIndexSize;
// read the segment index
read(p, buff);
long sip = getIntLong(buff, 0);
if (ip < sip) {
h = m - 1;
} else {
long eip = getIntLong(buff, 4);
if (ip > eip) {
l = m + 1;
} else {
dataLen = getInt2(buff, 8);
dataPtr = getInt(buff, 10);
break;
}
}
}
// empty match interception
// System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
if (dataPtr < 0) {
return null;
}
// load and return the region data
final byte[] regionBuff = new byte[dataLen];
read(dataPtr, regionBuff);
return new String(regionBuff);
}
protected void read(int offset, byte[] buffer) throws IOException {
// check the in-memory buffer first
if (contentBuff != null) {
// @TODO: reduce data copying, directly decode the data ?
System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
return;
}
// read from the file handle
assert handle != null;
handle.seek(offset);
this.ioCount++;
int rLen = handle.read(buffer);
if (rLen != buffer.length) {
throw new IOException("incomplete read: read bytes should be " + buffer.length);
}
}
// --- static cache util function
public static Header loadHeader(RandomAccessFile handle) throws IOException {
handle.seek(0);
final byte[] buff = new byte[HeaderInfoLength];
handle.read(buff);
return new Header(buff);
}
public static Header loadHeaderFromFile(String dbPath) throws IOException {
RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
return loadHeader(handle);
}
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
handle.seek(HeaderInfoLength);
int len = VectorIndexRows * VectorIndexCols * SegmentIndexSize;
final byte[] buff = new byte[len];
int rLen = handle.read(buff);
if (rLen != len) {
throw new IOException("incomplete read: read bytes should be " + len);
}
return buff;
}
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
return loadVectorIndex(handle);
}
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
handle.seek(0);
final byte[] buff = new byte[(int) handle.length()];
int rLen = handle.read(buff);
if (rLen != buff.length) {
throw new IOException("incomplete read: read bytes should be " + buff.length);
}
return buff;
}
public static byte[] loadContentFromFile(String dbPath) throws IOException {
RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
return loadContent(handle);
}
// --- End cache load util function
// --- static util method
/* get an int from a byte array start from the specified offset */
public static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset ] << 24) & 0xFF000000L)
);
}
public static int getInt(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FF)) |
((b[offset++] << 8) & 0x0000FF00) |
((b[offset++] << 16) & 0x00FF0000) |
((b[offset ] << 24) & 0xFF000000)
);
}
public static int getInt2(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset ] & 0x0000FF00)
);
}
/* long int to ip string */
public static String long2ip( long ip )
{
return String.valueOf((ip >> 24) & 0xFF) + '.' +
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
}
public static final byte[] shiftIndex = {24, 16, 8, 0};
/* check the specified ip address */
public static long checkIpAddr(String ip) throws Exception {
String[] ps = ip.split("\\.");
if (ps.length != 4) {
throw new Exception("invalid ip address `" + ip + "`");
}
long ipAddr = 0;
for (int i = 0; i < ps.length; i++) {
int val = Integer.parseInt(ps[i]);
if (val > 255) {
throw new Exception("ip part `"+ps[i]+"` should be less then 256");
}
ipAddr |= ((long) val << shiftIndex[i]);
}
return ipAddr & 0xFFFFFFFFL;
}
}
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或