“aefdbbc928772c563b93f68a6af50f59c713b36e”上不存在“mobile/src/operators/im2sequence_op.h”
提交 9f8721b7 编写于 作者: L liipx

add vendor files into repo

上级 2547102d
The MIT License (MIT)
Copyright (c) 2013 Stack Exchange
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
wmi
===
Package wmi provides a WQL interface to Windows WMI.
Note: It interfaces with WMI on the local machine, therefore it only runs on Windows.
// +build windows
package wmi
import (
"fmt"
"reflect"
"runtime"
"sync"
"github.com/go-ole/go-ole"
"github.com/go-ole/go-ole/oleutil"
)
// SWbemServices is used to access wmi. See https://msdn.microsoft.com/en-us/library/aa393719(v=vs.85).aspx
type SWbemServices struct {
//TODO: track namespace. Not sure if we can re connect to a different namespace using the same instance
cWMIClient *Client //This could also be an embedded struct, but then we would need to branch on Client vs SWbemServices in the Query method
sWbemLocatorIUnknown *ole.IUnknown
sWbemLocatorIDispatch *ole.IDispatch
queries chan *queryRequest
closeError chan error
lQueryorClose sync.Mutex
}
type queryRequest struct {
query string
dst interface{}
args []interface{}
finished chan error
}
// InitializeSWbemServices will return a new SWbemServices object that can be used to query WMI
func InitializeSWbemServices(c *Client, connectServerArgs ...interface{}) (*SWbemServices, error) {
//fmt.Println("InitializeSWbemServices: Starting")
//TODO: implement connectServerArgs as optional argument for init with connectServer call
s := new(SWbemServices)
s.cWMIClient = c
s.queries = make(chan *queryRequest)
initError := make(chan error)
go s.process(initError)
err, ok := <-initError
if ok {
return nil, err //Send error to caller
}
//fmt.Println("InitializeSWbemServices: Finished")
return s, nil
}
// Close will clear and release all of the SWbemServices resources
func (s *SWbemServices) Close() error {
s.lQueryorClose.Lock()
if s == nil || s.sWbemLocatorIDispatch == nil {
s.lQueryorClose.Unlock()
return fmt.Errorf("SWbemServices is not Initialized")
}
if s.queries == nil {
s.lQueryorClose.Unlock()
return fmt.Errorf("SWbemServices has been closed")
}
//fmt.Println("Close: sending close request")
var result error
ce := make(chan error)
s.closeError = ce //Race condition if multiple callers to close. May need to lock here
close(s.queries) //Tell background to shut things down
s.lQueryorClose.Unlock()
err, ok := <-ce
if ok {
result = err
}
//fmt.Println("Close: finished")
return result
}
func (s *SWbemServices) process(initError chan error) {
//fmt.Println("process: starting background thread initialization")
//All OLE/WMI calls must happen on the same initialized thead, so lock this goroutine
runtime.LockOSThread()
defer runtime.UnlockOSThread()
err := ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED)
if err != nil {
oleCode := err.(*ole.OleError).Code()
if oleCode != ole.S_OK && oleCode != S_FALSE {
initError <- fmt.Errorf("ole.CoInitializeEx error: %v", err)
return
}
}
defer ole.CoUninitialize()
unknown, err := oleutil.CreateObject("WbemScripting.SWbemLocator")
if err != nil {
initError <- fmt.Errorf("CreateObject SWbemLocator error: %v", err)
return
} else if unknown == nil {
initError <- ErrNilCreateObject
return
}
defer unknown.Release()
s.sWbemLocatorIUnknown = unknown
dispatch, err := s.sWbemLocatorIUnknown.QueryInterface(ole.IID_IDispatch)
if err != nil {
initError <- fmt.Errorf("SWbemLocator QueryInterface error: %v", err)
return
}
defer dispatch.Release()
s.sWbemLocatorIDispatch = dispatch
// we can't do the ConnectServer call outside the loop unless we find a way to track and re-init the connectServerArgs
//fmt.Println("process: initialized. closing initError")
close(initError)
//fmt.Println("process: waiting for queries")
for q := range s.queries {
//fmt.Printf("process: new query: len(query)=%d\n", len(q.query))
errQuery := s.queryBackground(q)
//fmt.Println("process: s.queryBackground finished")
if errQuery != nil {
q.finished <- errQuery
}
close(q.finished)
}
//fmt.Println("process: queries channel closed")
s.queries = nil //set channel to nil so we know it is closed
//TODO: I think the Release/Clear calls can panic if things are in a bad state.
//TODO: May need to recover from panics and send error to method caller instead.
close(s.closeError)
}
// Query runs the WQL query using a SWbemServices instance and appends the values to dst.
//
// dst must have type *[]S or *[]*S, for some struct type S. Fields selected in
// the query must have the same name in dst. Supported types are all signed and
// unsigned integers, time.Time, string, bool, or a pointer to one of those.
// Array types are not supported.
//
// By default, the local machine and default namespace are used. These can be
// changed using connectServerArgs. See
// http://msdn.microsoft.com/en-us/library/aa393720.aspx for details.
func (s *SWbemServices) Query(query string, dst interface{}, connectServerArgs ...interface{}) error {
s.lQueryorClose.Lock()
if s == nil || s.sWbemLocatorIDispatch == nil {
s.lQueryorClose.Unlock()
return fmt.Errorf("SWbemServices is not Initialized")
}
if s.queries == nil {
s.lQueryorClose.Unlock()
return fmt.Errorf("SWbemServices has been closed")
}
//fmt.Println("Query: Sending query request")
qr := queryRequest{
query: query,
dst: dst,
args: connectServerArgs,
finished: make(chan error),
}
s.queries <- &qr
s.lQueryorClose.Unlock()
err, ok := <-qr.finished
if ok {
//fmt.Println("Query: Finished with error")
return err //Send error to caller
}
//fmt.Println("Query: Finished")
return nil
}
func (s *SWbemServices) queryBackground(q *queryRequest) error {
if s == nil || s.sWbemLocatorIDispatch == nil {
return fmt.Errorf("SWbemServices is not Initialized")
}
wmi := s.sWbemLocatorIDispatch //Should just rename in the code, but this will help as we break things apart
//fmt.Println("queryBackground: Starting")
dv := reflect.ValueOf(q.dst)
if dv.Kind() != reflect.Ptr || dv.IsNil() {
return ErrInvalidEntityType
}
dv = dv.Elem()
mat, elemType := checkMultiArg(dv)
if mat == multiArgTypeInvalid {
return ErrInvalidEntityType
}
// service is a SWbemServices
serviceRaw, err := oleutil.CallMethod(wmi, "ConnectServer", q.args...)
if err != nil {
return err
}
service := serviceRaw.ToIDispatch()
defer serviceRaw.Clear()
// result is a SWBemObjectSet
resultRaw, err := oleutil.CallMethod(service, "ExecQuery", q.query)
if err != nil {
return err
}
result := resultRaw.ToIDispatch()
defer resultRaw.Clear()
count, err := oleInt64(result, "Count")
if err != nil {
return err
}
enumProperty, err := result.GetProperty("_NewEnum")
if err != nil {
return err
}
defer enumProperty.Clear()
enum, err := enumProperty.ToIUnknown().IEnumVARIANT(ole.IID_IEnumVariant)
if err != nil {
return err
}
if enum == nil {
return fmt.Errorf("can't get IEnumVARIANT, enum is nil")
}
defer enum.Release()
// Initialize a slice with Count capacity
dv.Set(reflect.MakeSlice(dv.Type(), 0, int(count)))
var errFieldMismatch error
for itemRaw, length, err := enum.Next(1); length > 0; itemRaw, length, err = enum.Next(1) {
if err != nil {
return err
}
err := func() error {
// item is a SWbemObject, but really a Win32_Process
item := itemRaw.ToIDispatch()
defer item.Release()
ev := reflect.New(elemType)
if err = s.cWMIClient.loadEntity(ev.Interface(), item); err != nil {
if _, ok := err.(*ErrFieldMismatch); ok {
// We continue loading entities even in the face of field mismatch errors.
// If we encounter any other error, that other error is returned. Otherwise,
// an ErrFieldMismatch is returned.
errFieldMismatch = err
} else {
return err
}
}
if mat != multiArgTypeStructPtr {
ev = ev.Elem()
}
dv.Set(reflect.Append(dv, ev))
return nil
}()
if err != nil {
return err
}
}
//fmt.Println("queryBackground: Finished")
return errFieldMismatch
}
// +build windows
/*
Package wmi provides a WQL interface for WMI on Windows.
Example code to print names of running processes:
type Win32_Process struct {
Name string
}
func main() {
var dst []Win32_Process
q := wmi.CreateQuery(&dst, "")
err := wmi.Query(q, &dst)
if err != nil {
log.Fatal(err)
}
for i, v := range dst {
println(i, v.Name)
}
}
*/
package wmi
import (
"bytes"
"errors"
"fmt"
"log"
"os"
"reflect"
"runtime"
"strconv"
"strings"
"sync"
"time"
"github.com/go-ole/go-ole"
"github.com/go-ole/go-ole/oleutil"
)
var l = log.New(os.Stdout, "", log.LstdFlags)
var (
ErrInvalidEntityType = errors.New("wmi: invalid entity type")
// ErrNilCreateObject is the error returned if CreateObject returns nil even
// if the error was nil.
ErrNilCreateObject = errors.New("wmi: create object returned nil")
lock sync.Mutex
)
// S_FALSE is returned by CoInitializeEx if it was already called on this thread.
const S_FALSE = 0x00000001
// QueryNamespace invokes Query with the given namespace on the local machine.
func QueryNamespace(query string, dst interface{}, namespace string) error {
return Query(query, dst, nil, namespace)
}
// Query runs the WQL query and appends the values to dst.
//
// dst must have type *[]S or *[]*S, for some struct type S. Fields selected in
// the query must have the same name in dst. Supported types are all signed and
// unsigned integers, time.Time, string, bool, or a pointer to one of those.
// Array types are not supported.
//
// By default, the local machine and default namespace are used. These can be
// changed using connectServerArgs. See
// http://msdn.microsoft.com/en-us/library/aa393720.aspx for details.
//
// Query is a wrapper around DefaultClient.Query.
func Query(query string, dst interface{}, connectServerArgs ...interface{}) error {
if DefaultClient.SWbemServicesClient == nil {
return DefaultClient.Query(query, dst, connectServerArgs...)
}
return DefaultClient.SWbemServicesClient.Query(query, dst, connectServerArgs...)
}
// A Client is an WMI query client.
//
// Its zero value (DefaultClient) is a usable client.
type Client struct {
// NonePtrZero specifies if nil values for fields which aren't pointers
// should be returned as the field types zero value.
//
// Setting this to true allows stucts without pointer fields to be used
// without the risk failure should a nil value returned from WMI.
NonePtrZero bool
// PtrNil specifies if nil values for pointer fields should be returned
// as nil.
//
// Setting this to true will set pointer fields to nil where WMI
// returned nil, otherwise the types zero value will be returned.
PtrNil bool
// AllowMissingFields specifies that struct fields not present in the
// query result should not result in an error.
//
// Setting this to true allows custom queries to be used with full
// struct definitions instead of having to define multiple structs.
AllowMissingFields bool
// SWbemServiceClient is an optional SWbemServices object that can be
// initialized and then reused across multiple queries. If it is null
// then the method will initialize a new temporary client each time.
SWbemServicesClient *SWbemServices
}
// DefaultClient is the default Client and is used by Query, QueryNamespace
var DefaultClient = &Client{}
// Query runs the WQL query and appends the values to dst.
//
// dst must have type *[]S or *[]*S, for some struct type S. Fields selected in
// the query must have the same name in dst. Supported types are all signed and
// unsigned integers, time.Time, string, bool, or a pointer to one of those.
// Array types are not supported.
//
// By default, the local machine and default namespace are used. These can be
// changed using connectServerArgs. See
// http://msdn.microsoft.com/en-us/library/aa393720.aspx for details.
func (c *Client) Query(query string, dst interface{}, connectServerArgs ...interface{}) error {
dv := reflect.ValueOf(dst)
if dv.Kind() != reflect.Ptr || dv.IsNil() {
return ErrInvalidEntityType
}
dv = dv.Elem()
mat, elemType := checkMultiArg(dv)
if mat == multiArgTypeInvalid {
return ErrInvalidEntityType
}
lock.Lock()
defer lock.Unlock()
runtime.LockOSThread()
defer runtime.UnlockOSThread()
err := ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED)
if err != nil {
oleCode := err.(*ole.OleError).Code()
if oleCode != ole.S_OK && oleCode != S_FALSE {
return err
}
}
defer ole.CoUninitialize()
unknown, err := oleutil.CreateObject("WbemScripting.SWbemLocator")
if err != nil {
return err
} else if unknown == nil {
return ErrNilCreateObject
}
defer unknown.Release()
wmi, err := unknown.QueryInterface(ole.IID_IDispatch)
if err != nil {
return err
}
defer wmi.Release()
// service is a SWbemServices
serviceRaw, err := oleutil.CallMethod(wmi, "ConnectServer", connectServerArgs...)
if err != nil {
return err
}
service := serviceRaw.ToIDispatch()
defer serviceRaw.Clear()
// result is a SWBemObjectSet
resultRaw, err := oleutil.CallMethod(service, "ExecQuery", query)
if err != nil {
return err
}
result := resultRaw.ToIDispatch()
defer resultRaw.Clear()
count, err := oleInt64(result, "Count")
if err != nil {
return err
}
enumProperty, err := result.GetProperty("_NewEnum")
if err != nil {
return err
}
defer enumProperty.Clear()
enum, err := enumProperty.ToIUnknown().IEnumVARIANT(ole.IID_IEnumVariant)
if err != nil {
return err
}
if enum == nil {
return fmt.Errorf("can't get IEnumVARIANT, enum is nil")
}
defer enum.Release()
// Initialize a slice with Count capacity
dv.Set(reflect.MakeSlice(dv.Type(), 0, int(count)))
var errFieldMismatch error
for itemRaw, length, err := enum.Next(1); length > 0; itemRaw, length, err = enum.Next(1) {
if err != nil {
return err
}
err := func() error {
// item is a SWbemObject, but really a Win32_Process
item := itemRaw.ToIDispatch()
defer item.Release()
ev := reflect.New(elemType)
if err = c.loadEntity(ev.Interface(), item); err != nil {
if _, ok := err.(*ErrFieldMismatch); ok {
// We continue loading entities even in the face of field mismatch errors.
// If we encounter any other error, that other error is returned. Otherwise,
// an ErrFieldMismatch is returned.
errFieldMismatch = err
} else {
return err
}
}
if mat != multiArgTypeStructPtr {
ev = ev.Elem()
}
dv.Set(reflect.Append(dv, ev))
return nil
}()
if err != nil {
return err
}
}
return errFieldMismatch
}
// ErrFieldMismatch is returned when a field is to be loaded into a different
// type than the one it was stored from, or when a field is missing or
// unexported in the destination struct.
// StructType is the type of the struct pointed to by the destination argument.
type ErrFieldMismatch struct {
StructType reflect.Type
FieldName string
Reason string
}
func (e *ErrFieldMismatch) Error() string {
return fmt.Sprintf("wmi: cannot load field %q into a %q: %s",
e.FieldName, e.StructType, e.Reason)
}
var timeType = reflect.TypeOf(time.Time{})
// loadEntity loads a SWbemObject into a struct pointer.
func (c *Client) loadEntity(dst interface{}, src *ole.IDispatch) (errFieldMismatch error) {
v := reflect.ValueOf(dst).Elem()
for i := 0; i < v.NumField(); i++ {
f := v.Field(i)
of := f
isPtr := f.Kind() == reflect.Ptr
if isPtr {
ptr := reflect.New(f.Type().Elem())
f.Set(ptr)
f = f.Elem()
}
n := v.Type().Field(i).Name
if !f.CanSet() {
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: "CanSet() is false",
}
}
prop, err := oleutil.GetProperty(src, n)
if err != nil {
if !c.AllowMissingFields {
errFieldMismatch = &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: "no such struct field",
}
}
continue
}
defer prop.Clear()
if prop.Value() == nil {
continue
}
switch val := prop.Value().(type) {
case int8, int16, int32, int64, int:
v := reflect.ValueOf(val).Int()
switch f.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
f.SetInt(v)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
f.SetUint(uint64(v))
default:
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: "not an integer class",
}
}
case uint8, uint16, uint32, uint64:
v := reflect.ValueOf(val).Uint()
switch f.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
f.SetInt(int64(v))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
f.SetUint(v)
default:
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: "not an integer class",
}
}
case string:
switch f.Kind() {
case reflect.String:
f.SetString(val)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
iv, err := strconv.ParseInt(val, 10, 64)
if err != nil {
return err
}
f.SetInt(iv)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
uv, err := strconv.ParseUint(val, 10, 64)
if err != nil {
return err
}
f.SetUint(uv)
case reflect.Struct:
switch f.Type() {
case timeType:
if len(val) == 25 {
mins, err := strconv.Atoi(val[22:])
if err != nil {
return err
}
val = val[:22] + fmt.Sprintf("%02d%02d", mins/60, mins%60)
}
t, err := time.Parse("20060102150405.000000-0700", val)
if err != nil {
return err
}
f.Set(reflect.ValueOf(t))
}
}
case bool:
switch f.Kind() {
case reflect.Bool:
f.SetBool(val)
default:
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: "not a bool",
}
}
case float32:
switch f.Kind() {
case reflect.Float32:
f.SetFloat(float64(val))
default:
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: "not a Float32",
}
}
default:
if f.Kind() == reflect.Slice {
switch f.Type().Elem().Kind() {
case reflect.String:
safeArray := prop.ToArray()
if safeArray != nil {
arr := safeArray.ToValueArray()
fArr := reflect.MakeSlice(f.Type(), len(arr), len(arr))
for i, v := range arr {
s := fArr.Index(i)
s.SetString(v.(string))
}
f.Set(fArr)
}
case reflect.Uint8:
safeArray := prop.ToArray()
if safeArray != nil {
arr := safeArray.ToValueArray()
fArr := reflect.MakeSlice(f.Type(), len(arr), len(arr))
for i, v := range arr {
s := fArr.Index(i)
s.SetUint(reflect.ValueOf(v).Uint())
}
f.Set(fArr)
}
default:
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: fmt.Sprintf("unsupported slice type (%T)", val),
}
}
} else {
typeof := reflect.TypeOf(val)
if typeof == nil && (isPtr || c.NonePtrZero) {
if (isPtr && c.PtrNil) || (!isPtr && c.NonePtrZero) {
of.Set(reflect.Zero(of.Type()))
}
break
}
return &ErrFieldMismatch{
StructType: of.Type(),
FieldName: n,
Reason: fmt.Sprintf("unsupported type (%T)", val),
}
}
}
}
return errFieldMismatch
}
type multiArgType int
const (
multiArgTypeInvalid multiArgType = iota
multiArgTypeStruct
multiArgTypeStructPtr
)
// checkMultiArg checks that v has type []S, []*S for some struct type S.
//
// It returns what category the slice's elements are, and the reflect.Type
// that represents S.
func checkMultiArg(v reflect.Value) (m multiArgType, elemType reflect.Type) {
if v.Kind() != reflect.Slice {
return multiArgTypeInvalid, nil
}
elemType = v.Type().Elem()
switch elemType.Kind() {
case reflect.Struct:
return multiArgTypeStruct, elemType
case reflect.Ptr:
elemType = elemType.Elem()
if elemType.Kind() == reflect.Struct {
return multiArgTypeStructPtr, elemType
}
}
return multiArgTypeInvalid, nil
}
func oleInt64(item *ole.IDispatch, prop string) (int64, error) {
v, err := oleutil.GetProperty(item, prop)
if err != nil {
return 0, err
}
defer v.Clear()
i := int64(v.Val)
return i, nil
}
// CreateQuery returns a WQL query string that queries all columns of src. where
// is an optional string that is appended to the query, to be used with WHERE
// clauses. In such a case, the "WHERE" string should appear at the beginning.
func CreateQuery(src interface{}, where string) string {
var b bytes.Buffer
b.WriteString("SELECT ")
s := reflect.Indirect(reflect.ValueOf(src))
t := s.Type()
if s.Kind() == reflect.Slice {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return ""
}
var fields []string
for i := 0; i < t.NumField(); i++ {
fields = append(fields, t.Field(i).Name)
}
b.WriteString(strings.Join(fields, ", "))
b.WriteString(" FROM ")
b.WriteString(t.Name())
b.WriteString(" " + where)
return b.String()
}
# CHANGELOG
## 2018-11
- DOING: english translation
- add -cleanup-test-database command-line arg
- fix -config arg load file error
- fix #87 RuleImplicitConversion value type mistach check bug
- fix #38 always true where condition check
- abandon stdin terminal interactive mod, which may seems like hangup
## 2018-10
- Fix SplitStatement mulitstatement eof bug #66
- Fix pretty func hangup issue #47
- Fix some foolish code spell error
- Use travis for CI
- Fix Go 1.8 defapth GOPATH compatible issue BUG #5
- 2018-10-20 开源先锋日(OSCAR)对外正式开源发布代码
## 2018-09
- 修复多个启发式建议不准确BUG,优化部分建议文案使得建议更清晰
- 基于TiDB Parser完善多个DDL类型语句的建议
- 新增lint report-type类型,支持Vim Plugin优化建议输出
- 更新整理项目文档,开源准备
- 2018-09-21 Gdevops SOAR首次对外进行技术分享宣传
## 2018-08
- 利用docker临时容器进行daily测试
- 添加main_test全功能回归测试
- 修复在测试中发现的问题
- mymysql合并MySQL8.0相关PR,修改vendor依赖
- 改善HeuristicRule中的文案
- 持续集成Vitess Parser的改进
- NewQuery4Audit结构体中引入TiDB Parser
- 通过TiAST完成大量与 DDL 相关的TODO
- 修改heuristic rules检查的返回值,提升拓展性
- 建议中引入Position,用于表示建议产生于SQL的位置
- 新增多个HeuristicRule
- Makefile中添加依赖检查,优化Makefile中逻辑,添加新功能
- 优化gometalinter性能,引入新的代码质量检测工具,提升代码质量
- 引入 retool 用于管理依赖的工具
- 优化 doc 文档
## 2018-07
- 补充文档,添加项目LOGO
- 改善代码质量提升测试覆盖度
- mymysql升级,支持MySQL 8.0
- 提供remove-comment小工具
- 提供索引重复检查小工具
- HeuristicRule新增RuleSpaceAfterDot
- 支持字符集和Collation不相同时的隐式数据类型转换的检查
## 2018-06
- 支持更多的SQL Rewrite规则
- 添加SQL执行超时限制
- 索引优化建议支持对约束的检查
- 修复数据采样中null值处理不正确的问题
- Explain支持last_query_cost
## 2018-05
- 添加数据采样功能
- 添加语句执行安全检查
- 支持DDL语法检查
- 支持DDL在测试环境的执行
- 支持隐式数据类型转换检查
- 支持索引去重
- 索引优化建议支持前缀索引
- 支持SQL Pretty输出
## 2018-04
- 支持语法检查
- 支持测试环境
- 支持MySQL原数据的获取
- 支持基于数据库环境信息给予索引优化建议
- 支持不依赖数据库原信息的简单索引优化建议
- 添加日志模块
- 引入配置文件
## 2018-03
- 基本架构设计
- 添加大量底层函数用于处理AST
- 添加Insert、Delete、Update转写成Select的基本函数
- 支持MySQL Explain信息输出
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at zhangliang3@xiaomi.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
Ask questions at [Gitter](https://gitter.im/xiaomi-dba/soar).
[Open an issue](https://github.com/xiaomi/soar/issues/new) to discuss your plans before doing any work on SOAR.
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
# This how we want to name the binary output
#
# use checkmake linter https://github.com/mrtazz/checkmake
# $ checkmake Makefile
#
BINARY=soar
GOPATH ?= $(shell go env GOPATH)
# Ensure GOPATH is set before running build process.
ifeq "$(GOPATH)" ""
$(error Please set the environment variable GOPATH before running `make`)
endif
PATH := ${GOPATH}/bin:$(PATH)
GCFLAGS=-gcflags "all=-trimpath=${GOPATH}"
LDFLAGS=-ldflags="-s -w"
# These are the values we want to pass for VERSION and BUILD
BUILD_TIME=`date +%Y%m%d%H%M`
COMMIT_VERSION=`git rev-parse HEAD`
# Add mysql version for testing `MYSQL_RELEASE=percona MYSQL_VERSION=5.7 make docker`
# MYSQL_RELEASE: mysql, percona, mariadb ...
# MYSQL_VERSION: latest, 8.0, 5.7, 5.6, 5.5 ...
# use mysql:latest as default
MYSQL_RELEASE := $(or ${MYSQL_RELEASE}, ${MYSQL_RELEASE}, mysql)
MYSQL_VERSION := $(or ${MYSQL_VERSION}, ${MYSQL_VERSION}, latest)
.PHONY: all
all: | fmt build
.PHONY: go_version_check
GO_VERSION_MIN=1.10
# Parse out the x.y or x.y.z version and output a single value x*10000+y*100+z (e.g., 1.9 is 10900)
# that allows the three components to be checked in a single comparison.
VER_TO_INT:=awk '{split(substr($$0, match ($$0, /[0-9\.]+/)), a, "."); print a[1]*10000+a[2]*100+a[3]}'
go_version_check:
@echo "\033[92mGo version check\033[0m"
@if test $(shell go version | $(VER_TO_INT) ) -lt \
$(shell echo "$(GO_VERSION_MIN)" | $(VER_TO_INT)); \
then printf "go version $(GO_VERSION_MIN)+ required, found: "; go version; exit 1; \
else echo "go version check pass"; fi
# Dependency check
.PHONY: deps
deps:
@echo "\033[92mDependency check\033[0m"
@bash ./deps.sh
# The retool tools.json is setup from retool-install.sh
retool sync
retool do gometalinter.v2 intall
# Code format
.PHONY: fmt
fmt: go_version_check
@echo "\033[92mRun gofmt on all source files ...\033[0m"
@echo "gofmt -l -s -w ..."
@ret=0 && for d in $$(go list -f '{{.Dir}}' ./... | grep -v /vendor/); do \
gofmt -l -s -w $$d/*.go || ret=$$? ; \
done ; exit $$ret
# Run golang test cases
.PHONY: test
test:
@echo "\033[92mRun all test cases ...\033[0m"
go test ./...
@echo "test Success!"
# Code Coverage
# colorful coverage numerical >=90% GREEN, <80% RED, Other YELLOW
.PHONY: cover
cover: test
@echo "\033[92mRun test cover check ...\033[0m"
go test -coverpkg=./... -coverprofile=coverage.data ./... | column -t
go tool cover -html=coverage.data -o coverage.html
go tool cover -func=coverage.data -o coverage.txt
@tail -n 1 coverage.txt | awk '{sub(/%/, "", $$NF); \
if($$NF < 80) \
{print "\033[91m"$$0"%\033[0m"} \
else if ($$NF >= 90) \
{print "\033[92m"$$0"%\033[0m"} \
else \
{print "\033[93m"$$0"%\033[0m"}}'
# Builds the project
build: fmt
@echo "\033[92mBuilding ...\033[0m"
@mkdir -p bin
@bash ./genver.sh
@ret=0 && for d in $$(go list -f '{{if (eq .Name "main")}}{{.ImportPath}}{{end}}' ./...); do \
b=$$(basename $${d}) ; \
go build ${GCFLAGS} -o bin/$${b} $$d || ret=$$? ; \
done ; exit $$ret
@echo "build Success!"
# Installs our project: copies binaries
install: build
@echo "\033[92mInstall ...\033[0m"
go install ./...
@echo "install Success!"
# Generate doc use -list* command
.PHONY: doc
doc: build
@echo "\033[92mAuto generate doc ...\033[0m"
./bin/soar -list-heuristic-rules > doc/heuristic.md
./bin/soar -list-rewrite-rules > doc/rewrite.md
./bin/soar -list-report-types > doc/report_type.md
# Add or change a heuristic rule
.PHONY: heuristic
heuristic: doc docker
@echo "\033[92mUpdate Heuristic rule golden files ...\033[0m"
go test github.com/XiaoMi/soar/advisor -v -update -run TestListHeuristicRules
go test github.com/XiaoMi/soar/advisor -v -update -run TestMergeConflictHeuristicRules
docker stop soar-mysql 2>/dev/null || true
# Update vitess vendor
.PHONY: vitess
vitess:
@echo "\033[92mUpdate vitess deps ...\033[0m"
govendor fetch -v vitess.io/vitess/...
# Update tidb vendor
.PHONY: tidb
tidb:
@echo "\033[92mUpdate tidb deps ...\033[0m"
govendor fetch -v github.com/pingcap/tidb/...
# make pingcap parser
.PHONY: pingcap-parser
pingcap-parser: tidb
@echo "\033[92mimporting pingcap parser ...\033[0m"
govendor fetch -v github.com/pingcap/parser/...
# Update all vendor
.PHONY: vendor
vendor: vitess pingcap-parser
# gometalinter
# 如果有不想改的lint问题可以使用metalinter.sh加黑名单
#@bash doc/example/metalinter.sh
.PHONY: lint
lint: build
@echo "\033[92mRun linter check ...\033[0m"
CGO_ENABLED=0 retool do gometalinter.v2 -j 1 --config doc/example/metalinter.json ./...
retool do revive -formatter friendly --exclude vendor/... -config doc/example/revive.toml ./...
retool do golangci-lint --tests=false run
@echo "gometalinter check your code is pretty good"
.PHONY: release
release: deps build
@echo "\033[92mCross platform building for release ...\033[0m"
@mkdir -p release
@for GOOS in darwin linux windows; do \
for GOARCH in amd64; do \
for d in $$(go list -f '{{if (eq .Name "main")}}{{.ImportPath}}{{end}}' ./...); do \
b=$$(basename $${d}) ; \
echo "Building $${b}.$${GOOS}-$${GOARCH} ..."; \
GOOS=$${GOOS} GOARCH=$${GOARCH} go build ${GCFLAGS} ${LDFLAGS} -v -o release/$${b}.$${GOOS}-$${GOARCH} $$d 2>/dev/null ; \
done ; \
done ;\
done
.PHONY: docker
docker:
@echo "\033[92mBuild mysql test enviorment\033[0m"
@docker stop soar-mysql 2>/dev/null || true
@echo "docker run --name soar-mysql $(MYSQL_RELEASE):$(MYSQL_VERSION)"
@docker run --name soar-mysql --rm -d \
-e MYSQL_ROOT_PASSWORD=1tIsB1g3rt \
-e MYSQL_DATABASE=sakila \
-p 3306:3306 \
-v `pwd`/doc/example/sakila.sql.gz:/docker-entrypoint-initdb.d/sakila.sql.gz \
$(MYSQL_RELEASE):$(MYSQL_VERSION)
@echo -n "waiting for sakila database initializing "
@while ! mysql -h 127.0.0.1 -u root sakila -p1tIsB1g3rt -NBe "do 1;" 2>/dev/null; do \
printf '.' ; \
sleep 1 ; \
done ; \
echo '.'
@echo "mysql test enviorment is ready!"
.PHONY: connect
connect:
mysql -h 127.0.0.1 -u root -p1tIsB1g3rt -c
.PHONY: main_test
main_test: install
@echo "\033[92mrunning main_test\033[0m"
@echo "soar -list-test-sqls | soar"
@./doc/example/main_test.sh
@echo "main_test Success!"
.PHONY: daily
daily: | deps fmt vendor docker cover doc lint release install main_test clean logo
@echo "\033[92mdaily build finished\033[0m"
# vendor, docker will cost long time, if all those are ready, daily-quick will much more fast.
.PHONY: daily-quick
daily-quick: | deps fmt cover doc lint logo
@echo "\033[92mdaily-quick build finished\033[0m"
.PHONY: logo
logo:
@echo "\033[93m"
@cat doc/images/logo.ascii
@echo "\033[m"
# Cleans our projects: deletes binaries
.PHONY: clean
clean:
@echo "\033[92mCleanup ...\033[0m"
go clean
@for GOOS in darwin linux windows; do \
for GOARCH in 386 amd64; do \
rm -f ${BINARY}.$${GOOS}-$${GOARCH} ;\
done ;\
done
rm -f ${BINARY} coverage.*
find . -name "*.log" -delete
git clean -fi
docker stop soar-mysql 2>/dev/null || true
Copyright 2018 Xiaomi, Inc. All Rights Reserved.
This product includes software developed by Xiaomi, Inc.
(http://www.mi.com/).
This product is licensed to you under the Apache License, Version 2.0
(the "License"). You may not use this product except in compliance with
the License.
# ![SOAR](https://raw.githubusercontent.com/XiaoMi/soar/master/doc/images/logo.png)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/xiaomi-dba/soar)
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://github.com/XiaoMi/soar/blob/master/LICENSE)
[![Go Report Card](https://goreportcard.com/badge/github.com/XiaoMi/soar)](https://goreportcard.com/report/github.com/XiaoMi/soar)
[![Build Status](https://travis-ci.org/XiaoMi/soar.svg?branch=master)](https://travis-ci.org/XiaoMi/soar)
[![GoDoc](https://godoc.org/github.com/XiaoMi/soar?status.svg)](https://godoc.org/github.com/XiaoMi/soar)
[文档](http://github.com/XiaoMi/soar/tree/master/doc) | [FAQ](http://github.com/XiaoMi/soar/blob/master/doc/FAQ.md) | [变更记录](http://github.com/XiaoMi/soar/blob/master/CHANGES.md) | [路线图](http://github.com/XiaoMi/soar/blob/master/doc/roadmap.md) | [English](http://github.com/XiaoMi/soar/blob/master/README_EN.md)
## SOAR
SOAR(SQL Optimizer And Rewriter)是一个对SQL进行优化和改写的自动化工具。 由小米人工智能与云平台的数据库团队开发与维护。
## 功能特点
* 跨平台支持(支持Linux, Mac环境,Windows环境理论上也支持,不过未全面测试)
* 目前只支持 MySQL 语法族协议的SQL优化
* 支持基于启发式算法的语句优化
* 支持复杂查询的多列索引优化(UPDATE, INSERT, DELETE, SELECT)
* 支持EXPLAIN信息丰富解读
* 支持SQL指纹、压缩和美化
* 支持同一张表多条ALTER请求合并
* 支持自定义规则的SQL改写
## 快速入门
* [安装使用](http://github.com/XiaoMi/soar/blob/master/doc/install.md)
* [体系架构](http://github.com/XiaoMi/soar/blob/master/doc/structure.md)
* [配置文件](http://github.com/XiaoMi/soar/blob/master/doc/config.md)
* [常用命令](http://github.com/XiaoMi/soar/blob/master/doc/cheatsheet.md)
* [产品对比](http://github.com/XiaoMi/soar/blob/master/doc/comparison.md)
* [路线图](http://github.com/XiaoMi/soar/blob/master/doc/roadmap.md)
## 交流与反馈
* 欢迎通过Github Issues提交问题报告与建议
* QQ群: 779359816(满) 758940447(新)
* [Gitter](https://gitter.im/xiaomi-dba/soar) 推荐
![xiaomi_sa](https://raw.githubusercontent.com/XiaoMi/soar/master/doc/images/xiaomi_sa.png)
## License
[Apache License 2.0](https://github.com/XiaoMi/soar/blob/master/LICENSE).
# ![SOAR](https://raw.githubusercontent.com/XiaoMi/soar/master/doc/images/logo.png)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/xiaomi-dba/soar)
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://github.com/XiaoMi/soar/blob/master/LICENSE)
[![Go Report Card](https://goreportcard.com/badge/github.com/XiaoMi/soar)](https://goreportcard.com/report/github.com/XiaoMi/soar)
[![Build Status](https://travis-ci.org/XiaoMi/soar.svg?branch=master)](https://travis-ci.org/XiaoMi/soar)
[![GoDoc](https://godoc.org/github.com/XiaoMi/soar?status.svg)](https://godoc.org/github.com/XiaoMi/soar)
[Docs](http://github.com/XiaoMi/soar/tree/master/doc) | [FAQ](http://github.com/XiaoMi/soar/blob/master/doc/FAQ_en.md) | [中文](http://github.com/XiaoMi/soar/blob/master/README.md)
## SOAR
SOAR (SQL Optimizer And Rewriter) is a tool, which can help SQL optimization and rewrite. It's developed and maintained by the DBA Team of Xiaomi AI&Cloud.
## Features
* Cross-platform support, such as Linux, Mac, and Windows
* Support Heuristic Rules Suggestion
* Support Complicate SQL Indexing Optimize
* Support EXPLAIN analyze for query plan
* Support SQL fingerprint, compress and built-in pretty print
* Support merge multi ALTER query into one SQL
* Support self-config rewrite rules from SQL Rewrite
* Suggestions were written in Chinese. But SOAR also gives many tools, which can be used without understanding Chinese.
## QuickStart
* [Install](http://github.com/XiaoMi/soar/blob/master/doc/install_en.md)
* [CheatSheet](http://github.com/XiaoMi/soar/blob/master/doc/cheatsheet_en.md)
* [Related works](http://github.com/XiaoMi/soar/blob/master/doc/comparison_en.md)
## Communication
* GitHub issues: bug reports, usage issues, feature requests
* [Gitter](https://gitter.im/xiaomi-dba/soar)
* IM QQ Group: 779359816
## License
[Apache License 2.0](https://github.com/XiaoMi/soar/blob/master/LICENSE).
/*
* Copyright 2018 Xiaomi, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package advisor contain heuristic rules, index rules and explain translator.
package advisor
/*
* Copyright 2018 Xiaomi, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package advisor
import (
"fmt"
"strings"
"github.com/XiaoMi/soar/common"
"github.com/XiaoMi/soar/database"
)
var explainRuleID int
// [EXP.XXX]Rule
var explainRules map[string]Rule
// [table_name]"suggest text"
var tablesSuggests map[string][]string
/*
var explainIgnoreTables = []string{
"dual",
"",
}
*/
// explain建议的形式
// Item: EXP.XXX
// Severity: L[0-8]
// Summary: full table scan, not use index, full index scan...
// Content: XX TABLE xxx
//
func checkExplainSelectType(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if len(common.Config.ExplainWarnSelectType) == 1 {
if common.Config.ExplainWarnSelectType[0] == "" {
return
}
} else if len(common.Config.ExplainWarnSelectType) < 1 {
return
}
if exp.ExplainFormat == database.JSONFormatExplain {
// TODO
// JSON 形式遍历分析不方便,转成 Row 格式也没有 SelectType 暂不处理
return
}
for _, v := range common.Config.ExplainWarnSelectType {
for _, row := range exp.ExplainRows {
if row.SelectType == v && v != "" {
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("SelectType:%s", row.SelectType))
}
}
}
}
// 用户可以设置AccessType的建议级别,匹配到的查询会给出建议
func checkExplainAccessType(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if len(common.Config.ExplainWarnAccessType) == 1 {
if common.Config.ExplainWarnAccessType[0] == "" {
return
}
} else if len(common.Config.ExplainWarnAccessType) < 1 {
return
}
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
// JSON形式遍历分析不方便,转成Row格式统一处理
rows = database.ConvertExplainJSON2Row(exp.ExplainJSON)
}
for _, v := range common.Config.ExplainWarnAccessType {
for _, row := range rows {
if row.AccessType == v && v != "" {
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("Scalability:%s", row.Scalability))
}
}
}
}
// TODO:
/*
func checkExplainPossibleKeys(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if common.Config.ExplainMinPossibleKeys == 0 {
return
}
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
// JSON形式遍历分析不方便,转成Row格式统一处理
rows = database.ConvertExplainJSON2Row(exp.ExplainJSON)
}
for _, row := range rows {
if len(row.PossibleKeys) < common.Config.ExplainMinPossibleKeys {
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("PossibleKeys:%d < %d",
len(row.PossibleKeys), common.Config.ExplainMinPossibleKeys))
}
}
}
*/
// TODO:
/*
func checkExplainKeyLen(exp *database.ExplainInfo) {
}
*/
// TODO:
/*
func checkExplainKey(exp *database.ExplainInfo) {
// 小于最小使用试用key数量
//return intval($explainResult) < intval($userCond);
//explain-min-keys int
}
*/
func checkExplainRef(exp *database.ExplainInfo) {
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
// JSON形式遍历分析不方便,转成Row格式统一处理
rows = database.ConvertExplainJSON2Row(exp.ExplainJSON)
}
for i, row := range rows {
if strings.Join(row.Ref, "") == "NULL" || strings.Join(row.Ref, "") == "" {
if i == 0 && len(rows) > 1 {
continue
}
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("Ref:null"))
}
}
}
func checkExplainRows(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if common.Config.ExplainMaxRows <= 0 {
return
}
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
// JSON形式遍历分析不方便,转成Row格式统一处理
rows = database.ConvertExplainJSON2Row(exp.ExplainJSON)
}
for _, row := range rows {
if row.Rows >= common.Config.ExplainMaxRows {
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("Rows:%d", row.Rows))
}
}
}
// TODO:
/*
func checkExplainExtra(exp *database.ExplainInfo) {
// 包含用户配置的逗号分隔关键词之一则提醒
// return self::contains($explainResult, $userCond);
// explain-warn-extra []string
}
*/
func checkExplainFiltered(exp *database.ExplainInfo) {
// 判断是否跳过不检查
if common.Config.ExplainMaxFiltered <= 0.001 {
return
}
rows := exp.ExplainRows
if exp.ExplainFormat == database.JSONFormatExplain {
// JSON形式遍历分析不方便,转成Row格式统一处理
rows = database.ConvertExplainJSON2Row(exp.ExplainJSON)
}
for i, row := range rows {
if i == 0 && len(rows) > 1 {
continue
}
if row.Filtered >= common.Config.ExplainMaxFiltered {
tablesSuggests[row.TableName] = append(tablesSuggests[row.TableName], fmt.Sprintf("Filtered:%.2f%s", row.Filtered, "%"))
}
}
}
// ExplainAdvisor 基于explain信息给出建议
func ExplainAdvisor(exp *database.ExplainInfo) map[string]Rule {
common.Log.Debug("ExplainAdvisor SQL: %v", exp.SQL)
explainRuleID = 0
explainRules = make(map[string]Rule)
tablesSuggests = make(map[string][]string)
checkExplainSelectType(exp)
checkExplainAccessType(exp)
checkExplainFiltered(exp)
checkExplainRef(exp)
checkExplainRows(exp)
// 打印explain table
content := database.PrintMarkdownExplainTable(exp)
if common.Config.ShowWarnings {
content += "\n" + database.MySQLExplainWarnings(exp)
}
// 对explain table中各项难于理解的值做解释
cases := database.ExplainInfoTranslator(exp)
// 添加last_query_cost
if common.Config.ShowLastQueryCost {
content += "\n" + database.MySQLExplainQueryCost(exp)
}
if content != "" {
explainRules["EXP.000"] = Rule{
Item: "EXP.000",
Severity: "L0",
Summary: "Explain信息",
Content: content,
Case: cases,
Func: (*Query4Audit).RuleOK,
}
}
/*
for t, s := range tablesSuggests {
// 检查explain对应的表是否需要跳过,如dual,空表等
ig := false
for _, ti := range explainIgnoreTables {
if ti == t {
ig = true
}
}
if ig {
continue
}
ruleId := fmt.Sprintf("EXP.%03d", explainRuleId+1)
explainRuleId = explainRuleId + 1
explainRules[ruleId] = Rule{
Item: ruleId,
Severity: "L0",
Summary: fmt.Sprintf("表 `%s` 查询效率不高", t),
Content: fmt.Sprint("原因:", strings.Join(s, ",")),
Case: "",
Func: (*Query4Audit).RuleOK,
}
}
*/
return explainRules
}
// DigestExplainText 分析用户输入的EXPLAIN信息
func DigestExplainText(text string) {
// explain信息就不要显示完美了,美不美自己看吧。
common.Config.IgnoreRules = append(common.Config.IgnoreRules, "OK")
if !IsIgnoreRule("EXP.") {
explainInfo, err := database.ParseExplainText(text)
if err != nil {
common.Log.Error("main ParseExplainText Error: %v", err)
return
}
expSuggest := ExplainAdvisor(explainInfo)
_, output := FormatSuggest("", common.Config.ReportType, expSuggest)
if common.Config.ReportType == "html" {
fmt.Println(common.MarkdownHTMLHeader())
fmt.Println(common.Markdown2HTML(output))
} else {
fmt.Println(output)
}
}
}
/*
* Copyright 2018 Xiaomi, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package advisor
import (
"testing"
"github.com/XiaoMi/soar/common"
)
func TestDigestExplainText(t *testing.T) {
var text = `+----+-------------+---------+-------+---------------------------------------------------------+-------------------+---------+---------------------------+------+-------------+
| id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra |
+----+-------------+---------+-------+---------------------------------------------------------+-------------------+---------+---------------------------+------+-------------+
| 1 | SIMPLE | country | index | PRIMARY,country_id | country | 152 | NULL | 109 | Using index |
| 1 | SIMPLE | city | ref | idx_fk_country_id,idx_country_id_city,idx_all,idx_other | idx_fk_country_id | 2 | sakila.country.country_id | 2 | Using index |
+----+-------------+---------+-------+---------------------------------------------------------+-------------------+---------+---------------------------+------+-------------+`
common.Config.ReportType = "explain-digest"
err := common.GoldenDiff(func() { DigestExplainText(text) }, t.Name(), update)
if nil != err {
t.Fatal(err)
}
}
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
/*
* Copyright 2018 Xiaomi, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package ast is an interface for Abstract Syntax Tree parser
package ast
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
/*
* Copyright 2018 Xiaomi, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Package soar is the main program of SOAR
package main
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
package common
// -version输出信息
const (
Version = "2018-11-27 09:12:27 +0800 v0.8.1-112-gbbf012d"
Compile = "2018-11-27 11:33:40 +0800 by go version go1.10.4 darwin/amd64"
Branch = "master"
GitDirty = 35486
DevPath = "/Users/lpx/code/gopath/src/github.com/XiaoMi/soar"
)
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册