meta: support PostgreSQL as the meta engine (#542)

* support PostgreSQL as the meta engine

* fix test

* fix test

* add docs

* Update sql_test.go

docs/en/databases_for_metadata.md

@@ -64,6 +64,41 @@ $ sudo juicefs mount -d redis://192.168.1.6:6379/1 /mnt/jfs
 
 If you are interested, you can check [Redis Best Practices](redis_best_practices.md).
 
+## PostgreSQL
+
+PostgreSQL is one of the most popular open source relational databases in the world, and is often used as the preferred database for Web applications.
+
+You can easily buy a managed PostgreSQL database service on the cloud computing platform, or follow the [Quick Start Guide](https://www.postgresqltutorial.com/postgresql-getting-started/).
+
+Other PostgreSQL-compatible databases (such as CockRoachDB) can also be used as meta engine.
+
+### Create a file system
+
+When using PostgreSQL as the metadata storage engine, the following format is usually used to access the database:
+
+```shell
+postgres://[<username>:<password>@]<IP or Domain name>[:5432]/<database-name>[?parameters]
+```
+
+For example:
+
+```shell
+$ juicefs format --storage minio \
+	--bucket http://192.168.1.6:9000/jfs \
+	--access-key minioadmin \
+	--secret-key minioadmin \
+	postgres://user:password@192.168.1.6:5432/juicefs?sslmode=disable \
+	pics
+```
+
+For more connection parameters, [click here to view](https://pkg.go.dev/github.com/lib/pq#hdr-Connection_String_Parameters).
+
+### Mount a file system
+
+```shell
+$ sudo juicefs mount -d postgres://user:password@192.168.1.6:5432/juicefs?sslmode=disable /mnt/jfs
+```
+
 ## MySQL
 
 > MySQL is one of the most popular open source relational databases in the world, and is often used as the preferred database for Web applications.
@@ -175,7 +210,7 @@ $ sudo juicefs mount -d sqlite3:///home/herald/my-jfs.db /mnt/jfs/
 
 Since SQLite is a single-file database, usually only the host where the database is located can access it. Therefore, SQLite database is more suitable for stand-alone use. For multiple servers sharing the same file system, it is recommended to use databases such as Redis or MySQL.
 
-## TiDB
+## FoundationDB
 
 Coming soon...
 
@@ -183,6 +218,3 @@ Coming soon...
 
 Coming soon...
 
-## PostgreSQL
-
-Coming soon...

docs/zh_cn/databases_for_metadata.md

@@ -64,6 +64,41 @@ $ sudo juicefs mount -d redis://192.168.1.6:6379/1 /mnt/jfs
 
 如果你有兴趣，可以查看 [Redis 最佳实践](redis_best_practices.md)。
 
+## PostgreSQL
+
+PostgreSQL 是一款世界级的开源数据库，有完善的生态和丰富的应用场景，也可以用来作为 JuiceFS 的元数据引擎。
+
+许多云计算平台都提供托管的 PostgreSQL 数据库服务，也可以按照[使用向导](https://www.postgresqltutorial.com/postgresql-getting-started/)自己部署一个。
+
+其他跟 PostgreSQL 协议兼容的数据库（比如 CockroachDB 等) 也可以这样使用。
+
+### 创建文件系统
+
+使用 PostgreSQL 作为元数据引擎时，需要使用如下的格式来指定参数：
+
+```shell
+postgres://[<username>:<password>@]<IP or Domain name>[:5432]/<database-name>[?parameters]
+```
+
+比如:
+
+```shell
+$ juicefs format --storage minio \
+	--bucket http://192.168.1.6:9000/jfs \
+	--access-key minioadmin \
+	--secret-key minioadmin \
+	postgres://user:password@192.168.1.6:5432/juicefs?sslmode=disable \
+	pics
+```
+
+更多的连接参数，请 [参考这里](https://pkg.go.dev/github.com/lib/pq#hdr-Connection_String_Parameters).
+
+### 挂载文件系统
+
+```shell
+$ sudo juicefs mount -d postgres://user:password@192.168.1.6:5432/juicefs?sslmode=disable /mnt/jfs
+```
+
 ## MySQL
 
 > MySQL 是世界上最受欢迎的开源关系型数据库之一，常被作为 Web 应用程序的首选数据库。
@@ -175,14 +210,10 @@ $ sudo juicefs mount -d sqlite3:///home/herald/my-jfs.db /mnt/jfs/
 
 由于 SQLite 是一款单文件数据库，在不做特殊共享设置的情况下，通常只有数据库所在的主机可以访问它。因此，SQLite 数据库更适合单机使用，对于多台服务器共享同一文件系统的情况，建议使用 Redis 或 MySQL 等数据库。
 
-## TiDB
-
-即将推出......
-
 ## TiKV
 
 即将推出......
 
-## PostgreSQL
+## FoundationDB
 
-即将推出......
+即将推出......
\ No newline at end of file

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/juju/ratelimit v1.0.1
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/ks3sdklib/aws-sdk-go v0.0.0-20180820074416-dafab05ad142
+	github.com/lib/pq v1.8.0
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-sqlite3 v1.14.0
 	github.com/minio/cli v1.22.0

pkg/meta/interface.go

@@ -342,7 +342,12 @@ func NewClient(uri string, conf *Config) Meta {
 		if p < 0 {
 			logger.Fatalf("invalid uri: %s", uri)
 		}
-		m, err = newSQLMeta(uri[:p], uri[p+3:], conf)
+		driver := uri[:p]
+		if driver == "postgres" {
+			m, err = newSQLMeta(driver, uri, conf)
+		} else {
+			m, err = newSQLMeta(driver, uri[p+3:], conf)
+		}
 	}
 	if err != nil {
 		logger.Fatalf("Meta is not available: %s", err)

pkg/meta/redis_test.go

@@ -743,7 +743,11 @@ func testTruncateAndDelete(t *testing.T, m Meta) {
 		t.Fatalf("create file %s", st)
 	}
 	defer m.Unlink(ctx, 1, "f")
-	if st := m.Write(ctx, inode, 0, 100, Slice{1, 100, 0, 100}); st != 0 {
+	var cid uint64
+	if st := m.NewChunk(ctx, inode, 0, 100, &cid); st != 0 {
+		t.Fatalf("new chunk: %s", st)
+	}
+	if st := m.Write(ctx, inode, 0, 100, Slice{cid, 100, 0, 100}); st != 0 {
 		t.Fatalf("write file %s", st)
 	}
 	if st := m.Truncate(ctx, inode, 0, 200<<20, attr); st != 0 {

pkg/meta/sql.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
 	"github.com/mattn/go-sqlite3"
 	_ "github.com/mattn/go-sqlite3"
 	"xorm.io/xorm"
@@ -473,6 +474,9 @@ func (m *dbMeta) shouldRetry(err error) bool {
 	if err == nil {
 		return false
 	}
+	if _, ok := err.(syscall.Errno); ok {
+		return false
+	}
 	// TODO: add other retryable errors here
 	msg := err.Error()
 	switch m.engine.DriverName() {
@@ -481,6 +485,8 @@ func (m *dbMeta) shouldRetry(err error) bool {
 	case "mysql":
 		// MySQL, MariaDB or TiDB
 		return strings.Contains(msg, "try restarting transaction") || strings.Contains(msg, "try again later")
+	case "postgres":
+		return strings.Contains(msg, "current transaction is aborted") || strings.Contains(msg, "deadlock detected")
 	default:
 		return false
 	}
@@ -542,7 +548,7 @@ func (m *dbMeta) flushStats() {
 		newInodes := atomic.SwapInt64(&m.newInodes, 0)
 		if newSpace != 0 || newInodes != 0 {
 			err := m.txn(func(s *xorm.Session) error {
-				_, err := s.Exec("UPDATE jfs_counter SET value=value+ (CASE name WHEN 'usedSpace' THEN ? ELSE ? END) WHERE name='usedSpace' OR name='totalInodes' ", newSpace, newInodes)
+				_, err := s.Exec("UPDATE jfs_counter SET value=value+ CAST((CASE name WHEN 'usedSpace' THEN ? ELSE ? END) AS BIGINT) WHERE name='usedSpace' OR name='totalInodes' ", newSpace, newInodes)
 				return err
 			})
 			if err != nil {
@@ -765,7 +771,8 @@ func (m *dbMeta) SetAttr(ctx Context, inode Ino, set uint16, sugidclearmode uint
 func (m *dbMeta) appendSlice(s *xorm.Session, inode Ino, indx uint32, buf []byte) error {
 	var r sql.Result
 	var err error
-	if m.engine.DriverName() == "sqlite3" {
+	driver := m.engine.DriverName()
+	if driver == "sqlite3" || driver == "postgres" {
 		r, err = s.Exec("update jfs_chunk set slices=slices || ? where inode=? AND indx=?", buf, inode, indx)
 	} else {
 		r, err = s.Exec("update jfs_chunk set slices=concat(slices, ?) where inode=? AND indx=?", buf, inode, indx)
@@ -2303,6 +2310,10 @@ func (m *dbMeta) SetXattr(ctx Context, inode Ino, name string, value []byte) sys
 		var x = xattr{inode, name, value}
 		n, err := s.Insert(&x)
 		if err != nil || n == 0 {
+			if m.engine.DriverName() == "postgres" {
+				// cleanup failed session
+				_ = s.Rollback()
+			}
 			_, err = s.Update(&x, &xattr{inode, name, nil})
 		}
 		return err

pkg/meta/sql_test.go

@@ -74,6 +74,36 @@ func TestMySQLClient(t *testing.T) {
 	testCaseIncensi(t, m)
 }
 
+func TestPostgresQLClient(t *testing.T) {
+	m, err := newSQLMeta("postgres", "postgres://localhost:5432/test?sslmode=disable", &Config{})
+	if err != nil {
+		t.Skipf("create meta: %s", err)
+	}
+	m.engine.DropTables(&setting{})
+	m.engine.DropTables(&counter{})
+	m.engine.DropTables(&node{})
+	m.engine.DropTables(&edge{})
+	m.engine.DropTables(&symlink{})
+	m.engine.DropTables(&chunk{})
+	m.engine.DropTables(&chunkRef{})
+	m.engine.DropTables(&session{})
+	m.engine.DropTables(&sustained{})
+	m.engine.DropTables(&xattr{})
+	m.engine.DropTables(&delfile{})
+	m.engine.DropTables(&flock{})
+	m.engine.DropTables(&plock{})
+
+	testTruncateAndDelete(t, m)
+	testMetaClient(t, m)
+	testStickyBit(t, m)
+	testLocks(t, m)
+	testConcurrentWrite(t, m)
+	testCompaction(t, m)
+	testCopyFileRange(t, m)
+	m.conf.CaseInsensi = true
+	testCaseIncensi(t, m)
+}
+
 func TestStickyBitSQL(t *testing.T) {
 	tmp := tempFile(t)
 	defer os.Remove(tmp)