Scripts: add vme script

Makefile

@@ -169,7 +169,7 @@ emu: $(EMU)
 # extract verilog module from sim_top.v
 # usage: make vme VME_MODULE=Roq
 vme: $(SIM_TOP_V)
-	cd $(VME_HOME) && sbt "run -s $(VME_SOURCE)/build/XSSimTop.v -o $(VME_SOURCE)/build/extracted -m $(VME_MODULE)"
+	cd ./scripts/vme && sbt "run -s $(VME_SOURCE)/build/XSSimTop.v -o $(VME_SOURCE)/build/extracted -m $(VME_MODULE)"
 
 # usage: make phy_evaluate VME_MODULE=Roq REMOTE=100
 phy_evaluate: vme

scripts/vme/.gitignore

+### Project Specific stuff
+test_run_dir/*
+target/*
+project/target/*
+target/.history
+branch_record/*
+verilog/*
+log*
+.metals/*
+### XilinxISE template
+# intermediate build files
+*.bgn
+*.bit
+*.bld
+*.cmd_log
+*.drc
+*.ll
+*.lso
+*.msd
+*.msk
+*.ncd
+*.ngc
+*.ngd
+*.ngr
+*.pad
+*.par
+*.pcf
+*.prj
+*.ptwx
+*.rbb
+*.rbd
+*.stx
+*.syr
+*.twr
+*.twx
+*.unroutes
+*.ut
+*.xpi
+*.xst
+*_bitgen.xwbt
+*_envsettings.html
+*_map.map
+*_map.mrp
+*_map.ngm
+*_map.xrpt
+*_ngdbuild.xrpt
+*_pad.csv
+*_pad.txt
+*_par.xrpt
+*_summary.html
+*_summary.xml
+*_usage.xml
+*_xst.xrpt
+
+# project-wide generated files
+*.gise
+par_usage_statistics.html
+usage_statistics_webtalk.html
+webtalk.log
+webtalk_pn.xml
+
+# generated folders
+iseconfig/
+xlnx_auto_0_xdb/
+xst/
+_ngo/
+_xmsgs/
+### Eclipse template
+*.pydevproject
+.metadata
+.gradle
+bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+
+# Eclipse Core
+.project
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# CDT-specific
+.cproject
+
+# JDT-specific (Eclipse Java Development Tools)
+.classpath
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# TeXlipse plugin
+.texlipse
+### C template
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Debug files
+*.dSYM/
+### SBT template
+# Simple Build Tool
+# http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control
+
+target/
+lib_managed/
+src_managed/
+project/boot/
+.history
+.cache
+### Emacs template
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+### Vim template
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio
+
+*.iml
+
+## Directory-based project format:
+.idea/
+# if you remove the above rule, at least ignore the following:
+
+# User-specific stuff:
+# .idea/workspace.xml
+# .idea/tasks.xml
+# .idea/dictionaries
+
+# Sensitive or high-churn files:
+# .idea/dataSources.ids
+# .idea/dataSources.xml
+# .idea/sqlDataSources.xml
+# .idea/dynamic.xml
+# .idea/uiDesigner.xml
+
+# Gradle:
+# .idea/gradle.xml
+# .idea/libraries
+
+# Mongo Explorer plugin:
+# .idea/mongoSettings.xml
+
+## File-based project format:
+*.ipr
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+### C++ template
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+### OSX template
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+### Xcode template
+# Xcode
+#
+# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
+
+## Build generated
+build/
+DerivedData
+
+## Various settings
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+xcuserdata
+
+## Other
+*.xccheckout
+*.moved-aside
+*.xcuserstate
+### Scala template
+*.class
+*.log
+
+# sbt specific
+.cache
+.history
+.lib/
+dist/*
+target/
+lib_managed/
+src_managed/
+project/boot/
+project/plugins/project/
+
+# Scala-IDE specific
+.scala_dependencies
+.worksheet
+### Java template
+*.class
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+# vscode environment
+.vscode
+
+stale_outputs_checked
+

scripts/vme/README.md

+# vme
+## Usage: sbt "run [OPTION...]"  
++ **-s, --source**  
+    the verilog file generated by chisel, all in one file  
+    default: $NOOP_HOME/build/XSSimTop.v  
++ **-h, --help**  
+    print this help info  
++ **-o, --output**  
+    the place you want to store your extracted verilog  
+    default: $NOOP_HOME/build/extracted  
++ **-u, --usr**  
+    your name, will be used to name the output folder  
+    default: current user  
++ **-m, --modules**  
+    the top modules you would like to extract verilog from  
+    should always be the last argument  
+    default: IFU

scripts/vme/build.sbt

+scalaVersion := "2.13.4"
\ No newline at end of file

scripts/vme/project/build.properties

+sbt.version=1.4.2

scripts/vme/src/main/scala/getVerilogModules.scala

+package vme
+
+import scala.math._
+import scala.util._
+import scala.io.Source
+import scala.util.matching.Regex
+import scala.util.Using
+import scala.util.Try
+import scala._
+import scala.collection.mutable
+import java.io._
+import scala.language.postfixOps
+import sys.process._
+import sys._
+
+trait FileIOUtils {
+    // provides an interface to handle open and close when writing files
+    def writeToFile(f: String, action: (java.io.PrintWriter) => Unit) = {
+        val writer = new PrintWriter(new File(f))
+        action(writer)
+        writer.close()
+    }
+
+    def readFile[T](f: String, action: (scala.io.BufferedSource) => T): Try[T] = {
+        def checkExistence(file: String): String = {
+            val f = new File(file)
+            if (!f.exists()) {
+                println(s"file $f does not exist!")
+                sys.exit()
+            }
+            file
+        }
+        Using(Source.fromFile(checkExistence(f))) { s => action(s) }
+    }
+}
+
+class VerilogModuleExtractor() extends FileIOUtils {
+    //                            name
+    val modulePattern = "module ([\\w]+)\\(".r.unanchored
+    //                       type      name
+    val subMoudlePattern = "([\\w]+) ([\\w]+) \\((?: //.*)*\\Z".r.unanchored
+    val endMoudleIOPattern = "\\);".r.unanchored
+    val endMoudlePattern = "endmodule".r.unanchored
+
+    //                           (submoudle type, submoudle name)
+    type SubMoudleRecord = Tuple2[String, String]
+
+    //                        (content,          submodules)
+    type ModuleRecord = Tuple2[List[String], List[SubMoudleRecord]]
+    //                   name
+    type ModuleMap = Map[String, ModuleRecord]
+
+    def getLines(s: scala.io.BufferedSource): Iterator[String] = s.getLines()
+    
+    def makeRecord(s: Iterator[String]): ModuleMap = {
+        val m: ModuleMap = Map()
+        // called before we see the first line of a module
+        def processModule(firstLine: String, it: Iterator[String]): ModuleRecord = {
+            val content: List[String] = List(firstLine)
+            val submodules: List[SubMoudleRecord] = List()
+            def iter(cont: List[String], subm: List[SubMoudleRecord]): ModuleRecord = 
+                it.next() match {
+                    case l: String => l match {
+                        case endMoudlePattern() => (l :: cont, subm)
+                        case subMoudlePattern(ty, name) => {
+                            // println(s"submoudle $ty $name")
+                            iter(l :: cont, (ty, name) :: subm)
+                        }
+                        case _ => iter(l :: cont, subm)
+                    }
+                    case _ => {println("Should not reach here"); (cont, subm) }
+                }
+            val temp = iter(content, submodules)
+            (temp._1.reverse, temp._2)
+        }
+        def traverse(m: ModuleMap, it: Iterator[String]): ModuleMap = 
+            if (it.hasNext) {
+                it.next() match {
+                    case l: String => {
+                        // println(f"traversing $l")
+                        l match {
+                            case modulePattern(name) => {
+                                // println(f"get Module of name $name")
+                                traverse(m ++ Map(name -> processModule(l, it)), it)
+                            }
+                            case _ => {
+                                println(f"line $l is not a module definition")
+                                traverse(m, it)
+                            }
+                        }
+                    }
+                    case _ => traverse(m, it)
+                }
+            }
+            else m
+        
+        traverse(m, s)
+    }
+
+    def makeRecordFromFile(file: String): ModuleMap = {
+        readFile[ModuleMap](file, s => makeRecord(getLines(s))).get
+    }
+
+    def writeModuleToFile(name: String, record: ModuleRecord, dir: String) = {
+        val path = dir+name+".v"
+        println(f"Writing module $name%20s to $path")
+        writeToFile(path, w => {
+            record._1.foreach(l => w.write(f"$l\n"))
+        })
+    }
+
+    // get moudle definition of specified name
+    def getModule(name: String, m: ModuleMap): ModuleRecord = {
+        m(name)
+    }
+
+    def showModuleRecord(r: ModuleRecord) = {
+        val (content, submodules) = r
+        submodules.foreach {
+            case (t, n) => println(f"submoudle type: $t, submodule name: $n")
+        }
+        println("\nprinting module contents...")
+        content.foreach(println(_))
+    }
+    
+    // We first get records of all the modules and its submodule record
+    // Then we choose a module as the root node to traverse its submodule
+    def processFromModule(name: String, map: ModuleMap, outPath: String, doneSet: Set[String] = Set(), top: Tuple2[String, Boolean]): Unit = {
+        def printSRAMs(sub: List[SubMoudleRecord]) = {
+            sub map { t => t match {
+                case (ty, subn) if (ty contains "SRAM") => println(s"top module $name, sub module type $ty, name $subn")
+                case _ =>
+            }}
+        }
+        val (topName, isTop) = top
+        if (!map.contains(name)) {
+            println(s"${if (isTop) "chosen top" else s"submodule of ${topName},"} module $name does not exist!")
+            return
+        }
+        if (isTop) println(s"\nProcessing top module $name")
+        val r = map(name)
+        new File(outPath).mkdirs() // ensure the path exists
+        writeModuleToFile(name, r, outPath)
+        val submodules = r._2
+        // printSRAMs(submodules)
+        // DFS
+        val subTypesSet = submodules map (m => m._1) toSet
+        val nowMap = map - name
+        val nowSet = doneSet ++ subTypesSet
+        subTypesSet.foreach { s  => if (!doneSet.contains(s)) processFromModule(s, nowMap, outPath, nowSet, (if (isTop) name else topName, false)) }
+    }
+
+    def getDate: String = {
+        val d = java.time.LocalDate.now
+        d.toString().toCharArray().filterNot(_ == '-').mkString
+    }
+
+    def makePath(topModule: String, outDir: String , user: String = "glr"): String = {
+        (if (outDir.last == '/')
+            outDir
+        else
+            outDir+"/") + topModule + "/"
+            // outDir+"/") + getDate + "-" + user + "-" + topModule + "/"
+    }
+
+
+
+    def extract(src: String, topModule: String, outDir: String, user: String, mapp: Option[ModuleMap]): Unit = {
+        val useMap = mapp.getOrElse(makeRecordFromFile(src))
+        val path = makePath(topModule, outDir, user)
+        processFromModule(topModule, useMap, path, top=(topModule, true))
+    }
+
+    def extract(src: String, topModules: List[String], outDir: String, user: String): Unit = {
+        // avoid repeat
+        val mapp = makeRecordFromFile(src)
+        topModules.foreach(n => extract(src, n, outDir, user, Some(mapp)))
+    }
+}
+
+trait VMEArgParser {
+    type OptionMap = Map[String, Option[Any]]
+
+    val usage = """
+        Usage: sbt "run [OPTION...]"
+            -s, --source   the verilog file generated by chisel, all in one file
+                           default: $NOOP_HOME/build/XSSimTop.v
+            -h, --help     print this help info
+            -o, --output   the place you want to store your extracted verilog
+                           default: $NOOP_HOME/build/extracted
+            -u, --usr      your name, will be used to name the output folder
+                           default: current user
+            -m, --modules  the top modules you would like to extract verilog from
+                           should always be the last argument
+                           default: IFU
+      """
+
+    def parse(args: List[String]) = {
+        def nextOption(map: OptionMap, l: List[String]): OptionMap = {
+            def isSwitch(s : String)= (s(0) == '-')
+            l match {
+                case Nil => map
+                case ("--help" | "-h") :: tail => {
+                    println(usage)
+                    sys.exit()
+                    map
+                }
+                case ("--source" | "-s") :: file :: tail =>
+                    nextOption(map ++ Map("source" -> Some(file)), tail)
+                case ("--output" | "-o") :: path :: tail =>
+                    nextOption(map ++ Map("output" -> Some(path)), tail)
+                case ("--usr" | "-u") :: name :: tail =>
+                    nextOption(map ++ Map("usr" -> Some(name)), tail)
+                // this should always be the last argument, since it is length variable
+                case ("--modules" | "-m") :: m :: tail =>
+                    map ++ Map("modules" -> Some(m :: tail))
+                case s :: tail => {
+                    if (isSwitch(s)) println(s"unexpected argument $s")
+                    nextOption(map, tail)
+                }
+            }
+        }
+        nextOption(Map("source" -> None, "output" -> None, "usr" -> None, "modules" -> None), args)
+    }
+
+    
+
+    def wrapParams(args: Array[String]): (String, List[String], String, String) = {
+        val argL = args.toList
+        val paramMap = parse(argL)
+        (paramMap("source").map(_.asInstanceOf[String]).getOrElse(env("NOOP_HOME")+"/build/XSSimTop.v"),
+         paramMap("modules").map(_.asInstanceOf[List[String]]).getOrElse(List("IFU")),
+         paramMap("output").map(_.asInstanceOf[String]).getOrElse(env("NOOP_HOME")+"/build/extracted/"),
+         paramMap("usr").map(_.asInstanceOf[String]).getOrElse("whoami".!!))
+    }
+}
+
+object VMETest extends VMEArgParser {
+    def main(args: Array[String]): Unit = {
+        val vme = new VerilogModuleExtractor()
+        val (sourceFile, topModules, outTopDir, usr) = wrapParams(args)
+        vme.extract(sourceFile, topModules, outTopDir, usr)
+    }
+}