feat: import from remote file (#186)

* feat: add helper to download and remove remote files

* feat: add jsonl file check

* fix: spacing

* fix: comment

.gitignore

@@ -18,8 +18,11 @@
 /test*.csv
 /test*.env
 /test*.json
+/test*.jsonl
+*.jsonl
 *.json
 *.csv 
+temp
 
 # es log
 /elastic.log

appbase/common/utils.go

@@ -2,12 +2,16 @@ package common
 
 import (
 	"encoding/json"
-	"github.com/appbaseio/abc/log"
+	"fmt"
+	"io"
+	"net/http"
 	"net/url"
 	"os"
 	"os/exec"
 	"runtime"
 	"strings"
+
+	"github.com/appbaseio/abc/log"
 )
 
 // GetKeyForValue returns key for the given value
@@ -126,3 +130,49 @@ func Max(a, b int) int {
 	}
 	return b
 }
+
+// DefaultDownloadDirectory to download remote files
+var DefaultDownloadDirectory = "./temp"
+
+//DownloadFile create a local copy of the remote json file
+func DownloadFile(filepath string, url string) error {
+	log.Infoln("Downloading the file from remote URL:", url)
+	_, err := os.Stat(DefaultDownloadDirectory) //check if  directory exist
+	if os.IsNotExist(err) {
+
+		err = os.Mkdir(DefaultDownloadDirectory, 0755) //if not create, directory
+		if err != nil {
+			return err
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	_, err = io.Copy(out, resp.Body)
+	return err
+}
+
+// RemoveFile delete remote file
+func RemoveFile(fileName string) error {
+	log.Infoln("Deleting the temporary file:", fileName)
+	if err := os.Remove(fileName); err != nil {
+		log.Debugf(fmt.Sprintf("Unable to delete temporary file: %s", fileName), err)
+		return err
+	}
+	log.Infoln("file successfully deleted at path:", fileName)
+	return nil
+}

cmd/abc/appbase_import.go

@@ -205,8 +205,8 @@ func writeConfigFile(srcConfig map[string]interface{}, destConfig map[string]int
 			return "", nil, err
 		}
 	}
-	// check file path as source [json, csv]
-	if common.StringInSlice(srcConfig["_name_"].(string), []string{"json", "csv"}) {
+	// check file path as source [json, csv, jsonl]
+	if common.StringInSlice(srcConfig["_name_"].(string), []string{"json", "csv", "jsonl"}) {
 		err = common.IsFileValid(srcConfig["uri"].(string))
 		if err != nil {
 			return "", nil, err
@@ -355,4 +355,3 @@ func verifyConnectionsWithoutDestination(srcConfig map[string]interface{}) error
 	}
 	return nil
 }
-