Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Greenplum
Gpdb
提交
7118e8ac
G
Gpdb
项目概览
Greenplum
/
Gpdb
通知
7
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
G
Gpdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
7118e8ac
编写于
6月 10, 2020
作者:
W
Wen Lin
提交者:
GitHub
6月 10, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Revert "Add "FILL_MISSING_FIELDS" option for gpload." (#10280)
This reverts commit
87fef901
.
上级
683f767c
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
2 addition
and
46 deletion
+2
-46
gpMgmt/bin/gpload.py
gpMgmt/bin/gpload.py
+0
-5
gpMgmt/bin/gpload_test/gpload2/TEST.py
gpMgmt/bin/gpload_test/gpload2/TEST.py
+2
-12
gpMgmt/bin/gpload_test/gpload2/query39.ans
gpMgmt/bin/gpload_test/gpload2/query39.ans
+0
-16
gpMgmt/bin/gpload_test/gpload2/setup.ans
gpMgmt/bin/gpload_test/gpload2/setup.ans
+0
-8
gpMgmt/bin/gpload_test/gpload2/setup.sql
gpMgmt/bin/gpload_test/gpload2/setup.sql
+0
-5
未找到文件。
gpMgmt/bin/gpload.py
浏览文件 @
7118e8ac
...
...
@@ -112,7 +112,6 @@ valid_tokens = {
"quote"
:
{
'parse_children'
:
True
,
'parent'
:
"input"
},
"encoding"
:
{
'parse_children'
:
True
,
'parent'
:
"input"
},
"force_not_null"
:
{
'parse_children'
:
False
,
'parent'
:
"input"
},
"fill_missing_fields"
:
{
'parse_children'
:
False
,
'parent'
:
"input"
},
"error_limit"
:
{
'parse_children'
:
True
,
'parent'
:
"input"
},
"error_percent"
:
{
'parse_children'
:
True
,
'parent'
:
"input"
},
"error_table"
:
{
'parse_children'
:
True
,
'parent'
:
"input"
},
...
...
@@ -2366,10 +2365,6 @@ class gpload:
else
:
from_cols
=
self
.
from_columns
if
formatType
==
'csv'
or
formatType
==
'text'
:
if
self
.
getconfig
(
'gpload:input:fill_missing_fields'
,
bool
,
False
):
self
.
formatOpts
+=
'fill missing fields'
# If the 'reuse tables' option was specified we now try to find an
# already existing external table in the catalog which will match
# the one that we need to use. It must have identical attributes,
...
...
gpMgmt/bin/gpload_test/gpload2/TEST.py
浏览文件 @
7118e8ac
...
...
@@ -98,7 +98,7 @@ d = mkpath('config')
if
not
os
.
path
.
exists
(
d
):
os
.
mkdir
(
d
)
def
write_config_file
(
mode
=
'insert'
,
reuse_flag
=
''
,
columns_flag
=
'0'
,
mapping
=
'0'
,
portNum
=
'8081'
,
database
=
'reuse_gptest'
,
host
=
'localhost'
,
formatOpts
=
'text'
,
file
=
'data/external_file_01.txt'
,
table
=
'texttable'
,
format
=
'text'
,
delimiter
=
"'|'"
,
escape
=
''
,
quote
=
''
,
truncate
=
'False'
,
log_errors
=
None
,
error_limit
=
'0'
,
error_table
=
None
,
externalSchema
=
None
,
staging_table
=
None
,
fast_match
=
'false'
,
encoding
=
None
,
preload
=
True
,
fill
=
False
):
def
write_config_file
(
mode
=
'insert'
,
reuse_flag
=
''
,
columns_flag
=
'0'
,
mapping
=
'0'
,
portNum
=
'8081'
,
database
=
'reuse_gptest'
,
host
=
'localhost'
,
formatOpts
=
'text'
,
file
=
'data/external_file_01.txt'
,
table
=
'texttable'
,
format
=
'text'
,
delimiter
=
"'|'"
,
escape
=
''
,
quote
=
''
,
truncate
=
'False'
,
log_errors
=
None
,
error_limit
=
'0'
,
error_table
=
None
,
externalSchema
=
None
,
staging_table
=
None
,
fast_match
=
'false'
,
encoding
=
None
,
preload
=
True
):
f
=
open
(
mkpath
(
'config/config_file'
),
'w'
)
f
.
write
(
"VERSION: 1.0.0.1"
)
...
...
@@ -147,8 +147,6 @@ def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',
f
.
write
(
"
\n
- ESCAPE: "
+
escape
)
if
quote
:
f
.
write
(
"
\n
- QUOTE: "
+
quote
)
if
fill
:
f
.
write
(
"
\n
- FILL_MISSING_FIELDS: true"
)
f
.
write
(
"
\n
OUTPUT:"
)
f
.
write
(
"
\n
- TABLE: "
+
table
)
if
mode
:
...
...
@@ -445,7 +443,7 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
def
test_00_gpload_formatOpts_setup
(
self
):
"0 gpload setup"
for
num
in
range
(
1
,
40
):
for
num
in
range
(
1
,
39
):
f
=
open
(
mkpath
(
'query%d.sql'
%
num
),
'w'
)
f
.
write
(
"\! gpload -f "
+
mkpath
(
'config/config_file'
)
+
" -d reuse_gptest
\n
"
+
"\! gpload -f "
+
mkpath
(
'config/config_file'
)
+
" -d reuse_gptest
\n
"
)
f
.
close
()
...
...
@@ -755,14 +753,6 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
write_config_file
(
mode
=
'insert'
,
reuse_flag
=
'true'
,
fast_match
=
'false'
,
file
=
'data_file.txt'
,
error_table
=
"err_table"
,
error_limit
=
'1000'
,
preload
=
False
)
self
.
doTest
(
38
)
def
test_39_gpload_fill_missing_fields
(
self
):
"39 gpload fill missing fields"
file
=
mkpath
(
'setup.sql'
)
runfile
(
file
)
copy_data
(
'external_file_04.txt'
,
'data_file.txt'
)
write_config_file
(
mode
=
'insert'
,
reuse_flag
=
'false'
,
fast_match
=
'false'
,
file
=
'data_file.txt'
,
table
=
'texttable1'
,
error_limit
=
'1000'
,
fill
=
True
)
self
.
doTest
(
39
)
if
__name__
==
'__main__'
:
suite
=
unittest
.
TestLoader
().
loadTestsFromTestCase
(
GPLoad_FormatOpts_TestCase
)
runner
=
unittest
.
TextTestRunner
(
verbosity
=
2
)
...
...
gpMgmt/bin/gpload_test/gpload2/query39.ans
已删除
100644 → 0
浏览文件 @
683f767c
2020-06-02 14:55:41|INFO|gpload session started 2020-06-02 14:55:41
2020-06-02 14:55:46|INFO|setting schema 'public' for table 'texttable1'
2020-06-02 14:55:46|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
2020-06-02 14:55:46|INFO|running time: 5.27 seconds
2020-06-02 14:55:51|INFO|rows Inserted = 16
2020-06-02 14:55:51|INFO|rows Updated = 0
2020-06-02 14:55:51|INFO|data formatting errors = 0
2020-06-02 14:55:51|INFO|gpload succeeded
2020-06-02 14:55:51|INFO|gpload session started 2020-06-02 14:55:51
2020-06-02 14:55:56|INFO|setting schema 'public' for table 'texttable1'
2020-06-02 14:55:56|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
2020-06-02 14:55:56|INFO|running time: 5.09 seconds
2020-06-02 14:56:01|INFO|rows Inserted = 16
2020-06-02 14:56:01|INFO|rows Updated = 0
2020-06-02 14:56:01|INFO|data formatting errors = 0
2020-06-02 14:56:01|INFO|gpload succeeded
gpMgmt/bin/gpload_test/gpload2/setup.ans
浏览文件 @
7118e8ac
...
...
@@ -13,19 +13,11 @@ DROP TABLE
DROP TABLE IF EXISTS csvtable;
NOTICE: table "csvtable" does not exist, skipping
DROP TABLE
DROP TABLE IF EXISTS texttable1;
NOTICE: table "texttable1" does not exist, skipping
DROP TABLE
CREATE TABLE texttable (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision) DISTRIBUTED BY (n1);
CREATE TABLE
CREATE TABLE texttable1 (
s1 text, s2 text, s3 text, dt timestamp,
n1 smallint, n2 integer, n3 bigint, n4 decimal,
n5 numeric, n6 real, n7 double precision, n8 int) DISTRIBUTED BY (n1);
CREATE TABLE
CREATE TABLE csvtable (
year int, make text, model text, decription text, price decimal)
DISTRIBUTED BY (year);
...
...
gpMgmt/bin/gpload_test/gpload2/setup.sql
浏览文件 @
7118e8ac
...
...
@@ -9,15 +9,10 @@ CREATE SCHEMA test;
DROP
EXTERNAL
TABLE
IF
EXISTS
temp_gpload_staging_table
;
DROP
TABLE
IF
EXISTS
texttable
;
DROP
TABLE
IF
EXISTS
csvtable
;
DROP
TABLE
IF
EXISTS
texttable1
;
CREATE
TABLE
texttable
(
s1
text
,
s2
text
,
s3
text
,
dt
timestamp
,
n1
smallint
,
n2
integer
,
n3
bigint
,
n4
decimal
,
n5
numeric
,
n6
real
,
n7
double
precision
)
DISTRIBUTED
BY
(
n1
);
CREATE
TABLE
texttable1
(
s1
text
,
s2
text
,
s3
text
,
dt
timestamp
,
n1
smallint
,
n2
integer
,
n3
bigint
,
n4
decimal
,
n5
numeric
,
n6
real
,
n7
double
precision
,
n8
int
)
DISTRIBUTED
BY
(
n1
);
CREATE
TABLE
csvtable
(
year
int
,
make
text
,
model
text
,
decription
text
,
price
decimal
)
DISTRIBUTED
BY
(
year
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录