Commit 6d20e3b4 authored by Jonathan Mace's avatar Jonathan Mace

Add BDL compiler

parent 24bd2136
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>brown.tracingplane</groupId>
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>brown.tracingplane</groupId>
<artifactId>bdl-compiler</artifactId>
<packaging>jar</packaging>
......@@ -13,11 +13,185 @@
<version>1.0</version>
</parent>
<dependencies>
</dependencies>
<dependencies>
<dependency>
<groupId>com.lihaoyi</groupId>
<artifactId>fastparse_2.11</artifactId>
<version>0.4.2</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.11.8</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.10</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.10</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>1.48</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>19.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
</dependencies>
<build>
<plugins>
</plugins>
</build>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-eclipse-plugin</artifactId>
<configuration>
<downloadSources>true</downloadSources>
<downloadJavadocs>true</downloadJavadocs>
<projectnatures>
<projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
<projectnature>org.eclipse.jdt.core.javanature</projectnature>
</projectnatures>
<buildcommands>
<buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
</buildcommands>
<classpathContainers>
<classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
<classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
</classpathContainers>
<excludes>
<!-- in Eclipse, use scala-library, scala-compiler from the SCALA_CONTAINER
rather than POM <dependency> -->
<exclude>org.scala-lang:scala-library</exclude>
<exclude>org.scala-lang:scala-compiler</exclude>
</excludes>
<sourceIncludes>
<sourceInclude>**/*.scala</sourceInclude>
<sourceInclude>**/*.java</sourceInclude>
<sourceInclude>src/test/scala</sourceInclude>
</sourceIncludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.9.1</version>
<executions>
<execution>
<id>include-scala-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>src/main/scala</source>
</sources>
</configuration>
</execution>
<execution>
<id>include-scala-test-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>src/test/scala</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>appassembler-maven-plugin</artifactId>
<configuration>
<extraJvmArguments>-Xmx1000m
-Dlog4j.configuration="log4j-bbcompiler.properties"</extraJvmArguments>
<programs>
<program>
<mainClass>brown.tracingplane.bdl.compiler.BBC</mainClass>
<id>bbc</id>
</program>
</programs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>package-jar-with-dependencies</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../../resources/</outputDirectory>
<finalName>bbc</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>brown.tracingplane.bdl.compiler.BBC</mainClass>
</manifest>
</archive>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
package brown.tracingplane.bdl.compiler;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.log4j.PropertyConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import brown.tracingplane.bdl.compiler.JavaCompiler;
import brown.tracingplane.bdl.compiler.Ast.BaggageBuffersDeclaration;
public class BBC {
public static final String version = "0.1.alpha";
private static final Logger log = LoggerFactory.getLogger(BBC.class);
/** Command line parameters */
@Parameters(separators = "=")
public static class Settings {
@Parameter(names = { "--bag_path" },
description = "Specify the directory in which to search for imports. May be specified multiple times; directories will be searched in order. If not given, the current working directory is used.")
String bagPath = ".";
@Parameter(names = { "--version" }, description = "Show version info and exit.")
boolean version = false;
@Parameter(names = { "-h", "--help" }, help = true)
boolean help = false;
@Parameter(names = { "--java_out" }, description = "Output directory to generate Java source files")
String javaOut = null;
@Parameter(description = "file1 file2 ...")
List<String> files = new ArrayList<>();
}
public static void compile(Settings settings) throws CompileException {
Set<BaggageBuffersDeclaration> linked = Linker.link(settings);
if (settings.javaOut != null) {
new JavaCompiler().compile(settings.javaOut, linked);
}
}
public static void main(String[] args) {
PropertyConfigurator.configure(BBC.class.getClassLoader().getResourceAsStream("log4j-bbcompiler.properties"));
// Parse the args
Settings settings = new Settings();
JCommander jc = new JCommander(settings, args);
jc.setProgramName("bbc");
if (settings.help) {
jc.usage();
return;
}
if (settings.version) {
System.out.println("bbc " + version);
return;
}
if (settings.files.size() <= 0) {
System.out.println("Missing input file.");
return;
}
try {
compile(settings);
} catch (Exception e) {
log.error(e.getMessage());
}
}
}
\ No newline at end of file
package brown.tracingplane.bdl.compiler;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import brown.tracingplane.bdl.compiler.Ast.*;
public class CompileException extends Exception {
private static final long serialVersionUID = 1L;
public CompileException(String message, Object... formatArgs) {
super(String.format(message, formatArgs));
}
public CompileException(String message, Exception cause) {
this(cause, message);
}
public CompileException(Exception cause, String message, Object... formatArgs) {
super(String.format(message, formatArgs), cause);
}
public static CompileException sourceFileNotFound(String fileName) {
return new CompileException("Source file %s could not be found", fileName);
}
public static CompileException sourceFileNotReadable(String fileName, File file, IOException e) {
return new CompileException(e, "Source file %s (%s) unreadable: %s", fileName, file, e.getMessage());
}
public static CompileException importNotFound(String importName, File declaredIn, List<String> bagPath) {
return new CompileException("%s: import %s could not be found on bagPath %s", declaredIn, importName, FileUtils.joinBagPath(bagPath));
}
public static CompileException importNotReadable(File importFile, File declaredIn, String importedAs, IOException e) {
return new CompileException(e, "%s: import %s (%s) could not be read due to: %s", declaredIn, importedAs, importFile, e.getMessage());
}
public static CompileException sourceFileSyntaxError(String fileName, File file, Exception e) {
return new CompileException(e, "Source file %s (%s) syntax error: %s", fileName, file, e.getMessage());
}
public static CompileException syntaxError(Exception e) {
return new CompileException(e, "Syntax error: %s", e.getMessage());
}
public static CompileException importFileSyntaxError(File importFile, File declaredIn, String importedAs, Exception e) {
return new CompileException(e, "%s: import %s (%s) syntax error: %s", declaredIn, importedAs, importFile, e.getMessage());
}
public static CompileException recursiveImport(File first, File second) {
return new CompileException("%s: recursive import of %s", first, second);
}
public static CompileException duplicateDeclaration(File inputFile, String objectName) {
return new CompileException("%s: duplicate declaration of %s", inputFile, objectName);
}
public static CompileException invalidStructFieldType(File inputFile, String structName, StructFieldDeclaration field) {
return new CompileException("%s: %s is not a valid field type for struct %s (%s)", inputFile, field.fieldtype(), structName, field);
}
public static CompileException duplicateFieldDeclaration(File inputFile, String bagName, String fieldName, FieldDeclaration... fields) {
return new CompileException("%s: %s declares field named %s multiple times (%s)", inputFile, bagName, fieldName, StringUtils.join(fields, ", "));
}
public static CompileException duplicateStructFieldDeclaration(File inputFile, String bagName, String fieldName, StructFieldDeclaration... fields) {
return new CompileException("%s: %s declares field named %s multiple times (%s)", inputFile, bagName, fieldName, StringUtils.join(fields, ", "));
}
public static CompileException duplicateFieldDeclaration(File inputFile, String bagName, int fieldIndex, FieldDeclaration... fields) {
return new CompileException("%s: %s declares field index %d multiple times (%s)", inputFile, bagName, fieldIndex, StringUtils.join(fields, ", "));
}
public static CompileException unknownType(File inputFile, String bagName, FieldDeclaration declaration, UserDefinedType userDefined) {
return new CompileException("%s: %s declares field with unknown type %s (%s)", inputFile, bagName, userDefined, declaration);
}
public static CompileException unknownType(File inputFile, String structName, StructFieldDeclaration declaration, UserDefinedType userDefined) {
return new CompileException("%s: %s declares field with unknown type %s (%s)", inputFile, structName, userDefined, declaration);
}
public static CompileException fieldTypeNotValid(File inputFile, String bagName, FieldType fieldType) {
return new CompileException("%s: %s declares invalid field type %s", inputFile, bagName, fieldType);
}
}
package brown.tracingplane.bdl.compiler;
import java.util.Collection;
import brown.tracingplane.bdl.compiler.Ast.BaggageBuffersDeclaration;
import brown.tracingplane.bdl.compiler.Ast.ObjectDeclaration;
public interface Compiler {
public default void compile(String outputDir, Collection<BaggageBuffersDeclaration> bbDecls) {
bbDecls.forEach(decl -> compile(outputDir, decl));
}
public default void compile(String outputDir, BaggageBuffersDeclaration bbDecl) {
bbDecl.getObjectDeclarations().forEach(objectDecl -> compile(outputDir, objectDecl));
}
public void compile(String outputDir, ObjectDeclaration objectDecl);
}
package brown.tracingplane.bdl.compiler;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.google.common.collect.Lists;
/** Useful methods to do with reading files */
public class FileUtils {
private FileUtils() {}
public static List<String> splitBagPath(String bagPath) {
return Lists.<String>newArrayList(StringUtils.split(bagPath, File.pathSeparator));
}
public static String joinBagPath(List<String> bagPathElements) {
return StringUtils.join(bagPathElements, File.pathSeparator);
}
/**
* Simply gets the File object for the file with the specified name, but also checks if it exists and is readable
* @param fileName the name of the file to get
* @return a {@link File} object for this fileName or null if it could not be found
*/
public static File getFile(String fileName) {
File f = new File(fileName).getAbsoluteFile();
if (f.exists() && f.isFile() && f.canRead()) {
return f;
} else {
return null;
}
}
/**
* Search several directories for a file with the specified filename. Ignores files if they are not readable.
*
* @param fileName the name of the file to search for
* @param directories directories to search
* @return a {@link File} object for this fileName, or null if no file could be found in any of the directories
*/
public static File findFile(String fileName, List<String> directories) {
for (String directory : directories) {
File f = new File(directory, fileName);
if (f.exists() && f.isFile() && f.canRead()) {
return f;
}
}
return null;
}
/**
* Read the entire contents of a file as a String
*
* @param file the file to read
* @return the contents of the file
* @throws IOException if the file cannot be read for some reason
*/
public static String readFully(File file) throws IOException {
FileInputStream fis = new FileInputStream(file);
byte[] data = new byte[(int) file.length()];
fis.read(data);
fis.close();
return new String(data, Charset.defaultCharset());
}
}
package brown.tracingplane.bdl.compiler;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import org.apache.commons.lang3.CharUtils;
import org.apache.commons.lang3.StringUtils;
import brown.tracingplane.bdl.compiler.JavaCompiler;
import brown.tracingplane.bdl.compiler.Parser;
import brown.tracingplane.bdl.compiler.Ast.BagDeclaration;
import brown.tracingplane.bdl.compiler.Ast.BaggageBuffersDeclaration;
public class JavaCompilerUtils {
/**
* Replaces characters preceded by underscores with uppercase version, eg:
*
* 'hello_world' => 'helloWorld' 'hello_World' => 'helloWorld' 'hello__world' => 'hello_World'
*/
public static String formatCamelCase(String name) {
StringBuilder formatted = new StringBuilder();
for (int i = 0; i < name.length(); i++) {
char ci = name.charAt(i);
if (i < name.length() - 1 && ci == '_' && CharUtils.isAscii(ci)) {
char cii = name.charAt(i + 1);
if (CharUtils.isAsciiAlphaLower(cii)) {
formatted.append(StringUtils.upperCase(String.valueOf(cii)));
i++;
}
} else {
formatted.append(name.charAt(i));
}
}
return formatted.toString();
}
public static String indent(String str) {
return indent(str, " ");
}
public static String indent(String str, String indentationString) {
String[] splits = StringUtils.split(str, "\n");
for (int i = 0; i < splits.length; i++) {
splits[i] = indentationString + splits[i];
}
return StringUtils.join(splits, "\n");
}
// Simple indentation of str based on occurrences of { and }. Does not account for comments
public static String formatIndentation(String str, String indentWith) {
str = StringUtils.strip(str);
String[] splits = StringUtils.splitPreserveAllTokens(str, "\n");
int depth = 0;
for (int i = 0; i < splits.length; i++) {
// Unindent leading }'s
String line = StringUtils.strip(splits[i]);
while (line.startsWith("}")) {
line = StringUtils.strip(line.substring(1));
depth--;
}
// Indent line to correct depth
splits[i] = StringUtils.repeat(indentWith, depth) + StringUtils.strip(splits[i]);
splits[i] = StringUtils.stripEnd(splits[i], null); // in case empty line
depth += StringUtils.countMatches(line, "{");
depth -= StringUtils.countMatches(line, "}");
}
return StringUtils.join(splits, "\n");
}
public static void writeOutputFile(String baseDir, String packageName, String className, String fileContents) throws FileNotFoundException {
File outDir = new File(baseDir);
if (packageName != null && packageName.length() > 0) {
String[] splits = StringUtils.split(packageName, ".");
for (String split : splits) {
outDir = new File(outDir, split);
}
}
outDir.mkdirs();
File outFile = new File(outDir, className + ".java");
PrintWriter out = new PrintWriter(outFile);
out.write(fileContents);
out.close();
}
public static void main(String[] args) throws Exception {
String text = "package edu.brown.test; \nbag MyBag {\n" +
"bool firstField = 1; int32 secondField = 3; string whoops = 2;}";
JavaCompiler compiler = new JavaCompiler();
BaggageBuffersDeclaration decl = Parser.parseBaggageBuffersFile(text);
Linker.fillPackageNames(decl);
BagDeclaration bagDecl = decl.getBagDeclarations().get(0);
compiler.compile(".", bagDecl);
// String compiled = compiler.compile(bagDecl);
// System.out.println(formatIndentation(compiled, " "));
}
}
package brown.tracingplane.bdl.example;
bag SimpleBag {
set<fixed64> ids = 1;
}
bag SimpleBag2 {
int32 first_field = 1;
string second_field = 2;
}
bag ExampleBag {
bool boolfield = 0;
int32 int32field = 1;
sint32 sint32field = 2;
fixed32 fixed32field = 3;
sfixed32 sfixed32field = 4;
int64 int64field = 5;
sint64 sint64field = 6;
fixed64 fixed64field = 7;
sfixed64 sfixed64field = 8;
string stringfield = 9;
bytes bytesfield = 10;
set<int32> int32set = 11;
set<string> stringset = 12;
SimpleBag simple_bag = 15;
Set<string> simple_bag_2 = 16;