plug expr evaluation into columns/tables; create Identifier for tracking expression ids; support single quoting in expressions; tweak string to number coercion; implement topo sorter for calc col eval

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/branches/exprs@1148 f203690c-595d-4dc9-a70b-905162fa7fd2

src/main/java/com/healthmarketscience/jackcess/Database.java

@@ -28,6 +28,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TimeZone;
 
+import com.healthmarketscience.jackcess.expr.EvalConfig;
 import com.healthmarketscience.jackcess.query.Query;
 import com.healthmarketscience.jackcess.impl.DatabaseImpl;
 import com.healthmarketscience.jackcess.util.ColumnValidatorFactory;
@@ -68,7 +69,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    * the default sort order for table columns.
    * @usage _intermediate_field_
    */
-  public static final Table.ColumnOrder DEFAULT_COLUMN_ORDER = 
+  public static final Table.ColumnOrder DEFAULT_COLUMN_ORDER =
     Table.ColumnOrder.DATA;
 
   /** system property which can be used to set the default TimeZone used for
@@ -91,7 +92,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    *  if unspecified.
    * @usage _general_field_
    */
-  public static final String RESOURCE_PATH_PROPERTY = 
+  public static final String RESOURCE_PATH_PROPERTY =
     "com.healthmarketscience.jackcess.resourcePath";
 
   /** (boolean) system property which can be used to indicate that the current
@@ -99,7 +100,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    *  {@code FileChannel.transferFrom})
    * @usage _intermediate_field_
    */
-  public static final String BROKEN_NIO_PROPERTY = 
+  public static final String BROKEN_NIO_PROPERTY =
     "com.healthmarketscience.jackcess.brokenNio";
 
   /** system property which can be used to set the default sort order for
@@ -107,23 +108,30 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    *  values.
    * @usage _intermediate_field_
    */
-  public static final String COLUMN_ORDER_PROPERTY = 
+  public static final String COLUMN_ORDER_PROPERTY =
     "com.healthmarketscience.jackcess.columnOrder";
 
   /** system property which can be used to set the default enforcement of
    * foreign-key relationships.  Defaults to {@code true}.
    * @usage _general_field_
    */
-  public static final String FK_ENFORCE_PROPERTY = 
+  public static final String FK_ENFORCE_PROPERTY =
     "com.healthmarketscience.jackcess.enforceForeignKeys";
 
   /** system property which can be used to set the default allow auto number
    * insert policy.  Defaults to {@code false}.
    * @usage _general_field_
    */
-  public static final String ALLOW_AUTONUM_INSERT_PROPERTY = 
+  public static final String ALLOW_AUTONUM_INSERT_PROPERTY =
     "com.healthmarketscience.jackcess.allowAutoNumberInsert";
 
+  /** system property which can be used to enable expression evaluation
+   * (currently experimental).  Defaults to {@code false}.
+   * @usage _general_field_
+   */
+  public static final String ENABLE_EXPRESSION_EVALUATION_PROPERTY =
+    "com.healthmarketscience.jackcess.enableExpressionEvaluation";
+
   /**
    * Enum which indicates which version of Access created the database.
    * @usage _general_class_
@@ -160,7 +168,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
     public String getFileExtension() { return _ext; }
 
     @Override
-    public String toString() { 
+    public String toString() {
       return name() + " [" + DatabaseImpl.getFileFormatDetails(this).getFormat() + "]";
     }
   }
@@ -201,7 +209,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    * flexible iteration of Tables.
    */
   public TableIterableBuilder newIterable();
-  
+
   /**
    * @param name User table name (case-insensitive)
    * @return The Table, or null if it doesn't exist (or is a system table)
@@ -264,7 +272,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    * occassional time when access to a system table is necessary.  Messing
    * with system tables can strip the paint off your house and give your whole
    * family a permanent, orange afro.  You have been warned.
-   * 
+   *
    * @param tableName Table name, may be a system table
    * @return The table, or {@code null} if it doesn't exist
    * @usage _intermediate_method_
@@ -360,14 +368,14 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    */
   public Map<String,Database> getLinkedDatabases();
 
-  
+
   /**
    * Returns {@code true} if this Database links to the given Table, {@code
    * false} otherwise.
    * @usage _general_method_
    */
   public boolean isLinkedTable(Table table) throws IOException;
-  
+
   /**
    * Gets currently configured TimeZone (always non-{@code null}).
    * @usage _intermediate_method_
@@ -430,7 +438,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    * {@link #ALLOW_AUTONUM_INSERT_PROPERTY} system property).  Note that
    * <i>enabling this feature should be done with care</i> to reduce the
    * chances of screwing up the database.
-   * 
+   *
    * @usage _intermediate_method_
    */
   public boolean isAllowAutoNumberInsert();
@@ -443,6 +451,11 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    */
   public void setAllowAutoNumberInsert(Boolean allowAutoNumInsert);
 
+  // FIXME, docme
+  public boolean isEvaluateExpressions();
+
+  public void setEvaluateExpressions(Boolean evaluateExpressions);
+
   /**
    * Gets currently configured ColumnValidatorFactory (always non-{@code null}).
    * @usage _intermediate_method_
@@ -457,7 +470,7 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    * @usage _intermediate_method_
    */
   public void setColumnValidatorFactory(ColumnValidatorFactory newFactory);
-  
+
   /**
    * Returns the FileFormat of this database (which may involve inspecting the
    * database itself).
@@ -466,4 +479,8 @@ public interface Database extends Iterable<Table>, Closeable, Flushable
    */
   public FileFormat getFileFormat() throws IOException;
 
+  /**
+   * Returns the EvalConfig for configuring expression evaluation.
+   */
+  public EvalConfig getEvalConfig();
 }

src/main/java/com/healthmarketscience/jackcess/JackcessException.java

@@ -23,9 +23,9 @@ import java.io.IOException;
  *
  * @author James Ahlborn
  */
-public class JackcessException extends IOException 
+public class JackcessException extends IOException
 {
-  private static final long serialVersionUID = 20131123L;  
+  private static final long serialVersionUID = 20131123L;
 
   public JackcessException(String message) {
     super(message);

src/main/java/com/healthmarketscience/jackcess/expr/EvalConfig.java

@@ -0,0 +1,32 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.expr;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public interface EvalConfig 
+{
+  public TemporalConfig getTemporalConfig();
+
+  public void setTemporalConfig(TemporalConfig temporal);
+
+  public void putCustomExpressionFunction(Function func);
+
+  public Function getCustomExpressionFunction(String name);
+}

src/main/java/com/healthmarketscience/jackcess/expr/EvalContext.java

@@ -17,7 +17,6 @@ limitations under the License.
 package com.healthmarketscience.jackcess.expr;
 
 import java.text.SimpleDateFormat;
-import java.util.Random;
 
 /**
  *
@@ -25,16 +24,15 @@ import java.util.Random;
  */
 public interface EvalContext 
 {
-  public Value.Type getResultType();
-
   public TemporalConfig getTemporalConfig();
 
   public SimpleDateFormat createDateFormat(String formatStr);
 
-  public Value getThisColumnValue();
+  public float getRandom(Integer seed);
 
-  public Value getRowValue(String collectionName, String objName,
-                           String colName);
+  public Value.Type getResultType();
 
-  public float getRandom(Integer seed);
+  public Value getThisColumnValue();
+
+  public Value getIdentifierValue(Identifier identifier);
 }

src/main/java/com/healthmarketscience/jackcess/expr/Expression.java

@@ -16,6 +16,8 @@ limitations under the License.
 
 package com.healthmarketscience.jackcess.expr;
 
+import java.util.Collection;
+
 /**
  *
  * @author James Ahlborn
@@ -27,4 +29,6 @@ public interface Expression
   public String toDebugString();
 
   public boolean isConstant();
+
+  public void collectIdentifiers(Collection<Identifier> identifiers);
 }

src/main/java/com/healthmarketscience/jackcess/expr/Identifier.java

@@ -0,0 +1,84 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.expr;
+
+import org.apache.commons.lang.ObjectUtils;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class Identifier 
+{
+  private final String _collectionName;
+  private final String _objectName;
+  private final String _propertyName;
+
+  public Identifier(String collectionName, String objectName, String propertyName)
+  {
+    _collectionName = collectionName;
+    _objectName = objectName;
+    _propertyName = propertyName;
+  }
+
+  public String getCollectionName() 
+  {
+    return _collectionName;
+  }
+
+  public String getObjectName() 
+  {
+    return _objectName;
+  }
+
+  public String getPropertyName() 
+  {
+    return _propertyName;
+  }
+
+  @Override
+  public int hashCode() {
+    return _objectName.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if(!(o instanceof Identifier)) {
+      return false;
+    }
+
+    Identifier oi = (Identifier)o;
+    
+    return (ObjectUtils.equals(_objectName, oi._objectName) &&
+            ObjectUtils.equals(_collectionName, oi._collectionName) &&
+            ObjectUtils.equals(_propertyName, oi._propertyName));
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    if(_collectionName != null) {
+      sb.append("[").append(_collectionName).append("].");
+    }
+    sb.append("[").append(_objectName).append("]");
+    if(_propertyName != null) {
+      sb.append(".[").append(_propertyName).append("]");
+    }
+    return sb.toString();
+  }
+
+}

src/main/java/com/healthmarketscience/jackcess/impl/BaseEvalContext.java

@@ -0,0 +1,197 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumMap;
+import java.util.Map;
+
+import com.healthmarketscience.jackcess.DataType;
+import com.healthmarketscience.jackcess.JackcessException;
+import com.healthmarketscience.jackcess.expr.EvalContext;
+import com.healthmarketscience.jackcess.expr.EvalException;
+import com.healthmarketscience.jackcess.expr.Expression;
+import com.healthmarketscience.jackcess.expr.Identifier;
+import com.healthmarketscience.jackcess.expr.TemporalConfig;
+import com.healthmarketscience.jackcess.expr.Value;
+import com.healthmarketscience.jackcess.impl.expr.BuiltinOperators;
+import com.healthmarketscience.jackcess.impl.expr.Expressionator;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public abstract class BaseEvalContext implements EvalContext
+{
+  /** map of all non-string data types */
+  private static final Map<DataType,Value.Type> TYPE_MAP =
+    new EnumMap<DataType,Value.Type>(DataType.class);
+
+  static {
+    TYPE_MAP.put(DataType.BOOLEAN,Value.Type.LONG);
+    TYPE_MAP.put(DataType.BYTE,Value.Type.LONG);
+    TYPE_MAP.put(DataType.INT,Value.Type.LONG);
+    TYPE_MAP.put(DataType.LONG,Value.Type.LONG);
+    TYPE_MAP.put(DataType.MONEY,Value.Type.DOUBLE);
+    TYPE_MAP.put(DataType.FLOAT,Value.Type.DOUBLE);
+    TYPE_MAP.put(DataType.DOUBLE,Value.Type.DOUBLE);
+    TYPE_MAP.put(DataType.SHORT_DATE_TIME,Value.Type.DATE_TIME);
+    TYPE_MAP.put(DataType.NUMERIC,Value.Type.BIG_DEC);
+    TYPE_MAP.put(DataType.BIG_INT,Value.Type.BIG_DEC);
+  }
+
+  private final DBEvalContext _dbCtx;
+  private Expression _expr;
+
+  protected BaseEvalContext(DBEvalContext dbCtx) {
+    _dbCtx = dbCtx;
+  }
+
+  void setExpr(Expressionator.Type exprType, String exprStr) {
+    _expr = new RawExpr(exprType, exprStr);
+  }
+
+  protected DatabaseImpl getDatabase() {
+    return _dbCtx.getDatabase();
+  }
+
+  public TemporalConfig getTemporalConfig() {
+    return _dbCtx.getTemporalConfig();
+  }
+
+  public SimpleDateFormat createDateFormat(String formatStr) {
+    return _dbCtx.createDateFormat(formatStr);
+  }
+
+  public float getRandom(Integer seed) {
+    return _dbCtx.getRandom(seed);
+  }
+
+  public Value.Type getResultType() {
+    throw new UnsupportedOperationException();
+  }
+
+  public Value getThisColumnValue() {
+    throw new UnsupportedOperationException();
+  }
+
+  public Value getIdentifierValue(Identifier identifier) {
+    throw new UnsupportedOperationException();
+  }
+
+  public Object eval() throws IOException {
+    try {
+      return _expr.eval(this);
+    } catch(Exception e) {
+      String msg = withErrorContext(e.getMessage());
+      throw new JackcessException(msg, e);
+    }
+  }
+
+  public void collectIdentifiers(Collection<Identifier> identifiers) {
+    _expr.collectIdentifiers(identifiers);
+  }
+
+  @Override
+  public String toString() {
+    return _expr.toString();
+  }
+
+  protected Value toValue(Object val, DataType dType) {
+    try {
+      val = ColumnImpl.toInternalValue(dType, val, getDatabase());
+      if(val == null) {
+        return BuiltinOperators.NULL_VAL;
+      }
+
+      Value.Type vType = toValueType(dType);
+      switch(vType) {
+      case STRING:
+        return BuiltinOperators.toValue(val.toString());
+      case DATE:
+      case TIME:
+      case DATE_TIME:
+        return BuiltinOperators.toValue(this, vType, (Date)val);
+      case LONG:
+        Integer i = ((val instanceof Integer) ? (Integer)val :
+                     ((Number)val).intValue());
+        return BuiltinOperators.toValue(i);
+      case DOUBLE:
+        Double d = ((val instanceof Double) ? (Double)val :
+                    ((Number)val).doubleValue());
+        return BuiltinOperators.toValue(d);
+      case BIG_DEC:
+        BigDecimal bd = ColumnImpl.toBigDecimal(val, getDatabase());
+        return BuiltinOperators.toValue(bd);
+      default:
+        throw new RuntimeException("Unexpected type " + vType);
+      }
+    } catch(IOException e) {
+      throw new EvalException("Failed converting value to type " + dType, e);
+    }
+  }
+
+  protected static Value.Type toValueType(DataType dType) {
+    Value.Type type = TYPE_MAP.get(dType);
+    return ((type == null) ? Value.Type.STRING : type);
+  }
+
+  protected abstract String withErrorContext(String msg);
+
+  private class RawExpr implements Expression
+  {
+    private final Expressionator.Type _exprType;
+    private final String _exprStr;
+
+    private RawExpr(Expressionator.Type exprType, String exprStr) {
+      _exprType = exprType;
+      _exprStr = exprStr;
+    }
+
+    private Expression getExpr() {
+      // when the expression is parsed we replace the raw version
+      Expression expr = Expressionator.parse(_exprType, _exprStr, _dbCtx);
+      _expr = expr;
+      return expr;
+    }
+
+    public Object eval(EvalContext ctx) {
+      return getExpr().eval(ctx);
+    }
+
+    public String toDebugString() {
+      return "<raw>{" + _exprStr + "}";
+    }
+
+    public boolean isConstant() {
+      return getExpr().isConstant();
+    }
+
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      getExpr().collectIdentifiers(identifiers);
+    }
+
+    @Override
+    public String toString() {
+      return _exprStr;
+    }
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/CalcColEvalContext.java

@@ -0,0 +1,65 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.io.IOException;
+
+import com.healthmarketscience.jackcess.expr.Value;
+import com.healthmarketscience.jackcess.impl.expr.Expressionator;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class CalcColEvalContext extends RowEvalContext
+{
+  private final ColumnImpl _col;
+
+  public CalcColEvalContext(ColumnImpl col) {
+    super(col.getDatabase());
+    _col = col;
+  }
+
+  CalcColEvalContext setExpr(String exprStr) {
+    setExpr(Expressionator.Type.EXPRESSION, exprStr);
+    return this;
+  }
+
+  @Override
+  protected TableImpl getTable() {
+    return _col.getTable();
+  }
+
+  @Override
+  public Value.Type getResultType() {
+    return toValueType(_col.getType());
+  }
+
+  public Object eval(Object[] row) throws IOException {
+    try {
+      setRow(row);
+      return eval();
+    } finally {
+      reset();
+    }
+  }
+
+  @Override
+  protected String withErrorContext(String msg) {
+    return _col.withErrorContext(msg);
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/CalculatedColumnUtil.java

@@ -135,10 +135,22 @@ class CalculatedColumnUtil
    */
   private static class CalcColImpl extends ColumnImpl
   {
+    private CalcColEvalContext _calcCol;
+
     CalcColImpl(InitArgs args) throws IOException {
       super(args);
     }
 
+    @Override
+    protected CalcColEvalContext getCalculationContext() {
+      return _calcCol;
+    }
+
+    @Override
+    protected void setCalcColEvalContext(CalcColEvalContext calcCol) {
+      _calcCol = calcCol;
+    }
+
     @Override
     public Object read(byte[] data, ByteOrder order) throws IOException {
       data = unwrapCalculatedValue(data);
@@ -167,10 +179,22 @@ class CalculatedColumnUtil
    */
   private static class CalcBooleanColImpl extends ColumnImpl
   {
+    private CalcColEvalContext _calcCol;
+
     CalcBooleanColImpl(InitArgs args) throws IOException {
       super(args);
     }
 
+    @Override
+    protected CalcColEvalContext getCalculationContext() {
+      return _calcCol;
+    }
+
+    @Override
+    protected void setCalcColEvalContext(CalcColEvalContext calcCol) {
+      _calcCol = calcCol;
+    }
+
     @Override
     public boolean storeInNullMask() {
       // calculated booleans are _not_ stored in null mask
@@ -201,10 +225,22 @@ class CalculatedColumnUtil
    */
   private static class CalcTextColImpl extends TextColumnImpl
   {
+    private CalcColEvalContext _calcCol;
+
     CalcTextColImpl(InitArgs args) throws IOException {
       super(args);
     }
 
+    @Override
+    protected CalcColEvalContext getCalculationContext() {
+      return _calcCol;
+    }
+
+    @Override
+    protected void setCalcColEvalContext(CalcColEvalContext calcCol) {
+      _calcCol = calcCol;
+    }
+
     @Override
     public short getLengthInUnits() {
       // the byte "length" includes the calculated field overhead.  remove
@@ -232,10 +268,22 @@ class CalculatedColumnUtil
    */
   private static class CalcMemoColImpl extends MemoColumnImpl
   {
+    private CalcColEvalContext _calcCol;
+
     CalcMemoColImpl(InitArgs args) throws IOException {
       super(args);
     }
 
+    @Override
+    protected CalcColEvalContext getCalculationContext() {
+      return _calcCol;
+    }
+
+    @Override
+    protected void setCalcColEvalContext(CalcColEvalContext calcCol) {
+      _calcCol = calcCol;
+    }
+
     @Override
     protected int getMaxLengthInUnits() {
       // the byte "length" includes the calculated field overhead.  remove
@@ -264,10 +312,22 @@ class CalculatedColumnUtil
    */
   private static class CalcNumericColImpl extends NumericColumnImpl
   {
+    private CalcColEvalContext _calcCol;
+
     CalcNumericColImpl(InitArgs args) throws IOException {
       super(args);
     }
 
+    @Override
+    protected CalcColEvalContext getCalculationContext() {
+      return _calcCol;
+    }
+
+    @Override
+    protected void setCalcColEvalContext(CalcColEvalContext calcCol) {
+      _calcCol = calcCol;
+    }
+
     @Override
     public byte getPrecision() {
       return (byte)getType().getMaxPrecision();

src/main/java/com/healthmarketscience/jackcess/impl/ColDefaultValueEvalContext.java

@@ -0,0 +1,41 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import com.healthmarketscience.jackcess.expr.Value;
+import com.healthmarketscience.jackcess.impl.expr.Expressionator;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class ColDefaultValueEvalContext extends ColEvalContext 
+{
+  public ColDefaultValueEvalContext(ColumnImpl col) {
+    super(col);
+  }
+
+  ColDefaultValueEvalContext setExpr(String exprStr) {
+    setExpr(Expressionator.Type.DEFAULT_VALUE, exprStr);
+    return this;
+  }
+
+  @Override
+  public Value.Type getResultType() {
+    return toValueType(getCol().getType());
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/ColEvalContext.java

@@ -0,0 +1,43 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+
+
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public abstract class ColEvalContext extends BaseEvalContext
+{
+  private final ColumnImpl _col;
+
+  public ColEvalContext(ColumnImpl col) {
+    super(col.getDatabase().getEvalContext());
+    _col = col;
+  }
+
+  protected ColumnImpl getCol() {
+    return _col;
+  }
+
+  @Override
+  protected String withErrorContext(String msg) {
+    return _col.withErrorContext(msg);
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/ColValidatorEvalContext.java

@@ -0,0 +1,84 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.io.IOException;
+
+import com.healthmarketscience.jackcess.Column;
+import com.healthmarketscience.jackcess.InvalidValueException;
+import com.healthmarketscience.jackcess.expr.Value;
+import com.healthmarketscience.jackcess.impl.expr.Expressionator;
+import com.healthmarketscience.jackcess.util.ColumnValidator;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class ColValidatorEvalContext extends ColEvalContext
+{
+  private String _helpStr;
+  private Object _val;
+
+  public ColValidatorEvalContext(ColumnImpl col) {
+    super(col);
+  }
+
+  ColValidatorEvalContext setExpr(String exprStr, String helpStr) {
+    setExpr(Expressionator.Type.FIELD_VALIDATOR, exprStr);
+    _helpStr = helpStr;
+    return this;
+  }
+
+  ColumnValidator toColumnValidator(ColumnValidator delegate) {
+    return new InternalColumnValidator(delegate) {
+      @Override
+      protected Object internalValidate(Column col, Object val)
+        throws IOException {
+        return ColValidatorEvalContext.this.validate(col, val);
+      }
+      @Override
+      protected void appendToString(StringBuilder sb) {
+        sb.append("expression=").append(ColValidatorEvalContext.this);
+      }
+    };
+  }
+
+  private void reset() {
+    _val = null;
+  }
+
+  @Override
+  public Value getThisColumnValue() {
+    return toValue(_val, getCol().getType());
+  }
+
+  private Object validate(Column col, Object val) throws IOException {
+    try {
+      _val = val;
+      Boolean result = (Boolean)eval();
+      // FIXME how to handle null?
+      if(!result) {
+        String msg = ((_helpStr != null) ? _helpStr :
+                      "Invalid column value '" + val + "'");
+        throw new InvalidValueException(withErrorContext(msg));
+      }
+      return result;
+    } finally {
+      reset();
+    }
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/ColumnImpl.java

@@ -189,6 +189,8 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
   private PropertyMap _props;
   /** Validator for writing new values */
   private ColumnValidator _validator = SimpleColumnValidator.INSTANCE;
+  /** default value generator */
+  private ColDefaultValueEvalContext _defValue;
 
   /**
    * @usage _advanced_method_
@@ -492,10 +494,38 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
     setColumnValidator(null);
 
     // next, initialize any "internal" (property defined) validators
-    initPropertiesValidator();
+    reloadPropertiesValidators();
   }
 
-  void initPropertiesValidator() throws IOException {
+  void reloadPropertiesValidators() throws IOException {
+
+    if(isAutoNumber()) {
+      // none of the props stuff applies to autonumber columns
+      return;
+    }
+
+    if(isCalculated()) {
+
+      CalcColEvalContext calcCol = null;
+
+      if(getDatabase().isEvaluateExpressions()) {
+
+        // init calc col expression evaluator
+        PropertyMap props = getProperties();
+        String calcExpr = (String)props.getValue(PropertyMap.EXPRESSION_PROP);
+        calcCol = new CalcColEvalContext(this).setExpr(calcExpr);
+      }
+
+      setCalcColEvalContext(calcCol);
+
+      // none of the remaining props stuff applies to calculated columns
+      return;
+    }
+
+    // discard any existing internal validators and re-compute them
+    // (essentially unwrap the external validator)
+    _validator = getColumnValidator();
+    _defValue = null;
 
     PropertyMap props = getProperties();
 
@@ -515,12 +545,34 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
     if(!allowZeroLen) {
       _validator = new NoZeroLenColValidator(_validator);
     }
+
+    // only check for props based exprs if this is enabled
+    if(!getDatabase().isEvaluateExpressions()) {
+      return;
+    }
+
+    String exprStr = PropertyMaps.getTrimmedStringProperty(
+        props, PropertyMap.VALIDATION_RULE_PROP);
+
+    if(exprStr != null) {
+      String helpStr = PropertyMaps.getTrimmedStringProperty(
+          props, PropertyMap.VALIDATION_TEXT_PROP);
+
+      _validator = new ColValidatorEvalContext(this)
+        .setExpr(exprStr, helpStr)
+        .toColumnValidator(_validator);
+    }
+
+    String defValueStr = PropertyMaps.getTrimmedStringProperty(
+        props, PropertyMap.DEFAULT_VALUE_PROP);
+    if(defValueStr != null) {
+      _defValue = new ColDefaultValueEvalContext(this)
+        .setExpr(defValueStr);
+    }
   }
 
   void propertiesUpdated() throws IOException {
-    // discard any existing internal validators and re-compute them
-    _validator = getColumnValidator();
-    initPropertiesValidator();
+    reloadPropertiesValidators();
   }
 
   public ColumnValidator getColumnValidator() {
@@ -1066,6 +1118,13 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
     return GUID_PATTERN.matcher(toCharSequence(value)).matches();
   }
 
+  /**
+   * Returns a default value for this column
+   */
+  public Object generateDefaultValue() throws IOException {
+    return ((_defValue != null) ? _defValue.eval() : null);
+  }
+
   /**
    * Passes the given obj through the currently configured validator for this
    * column and returns the result.
@@ -1074,6 +1133,17 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
     return _validator.validate(this, obj);
   }
 
+  /**
+   * Returns the context used to manage calculated column values.
+   */
+  protected CalcColEvalContext getCalculationContext() {
+    throw new UnsupportedOperationException();
+  }
+
+  protected void setCalcColEvalContext(CalcColEvalContext calcCol) {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * Serialize an Object into a raw byte value for this column in little
    * endian order
@@ -1411,7 +1481,9 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
       .append("length", _columnLength)
       .append("variableLength", _variableLength);
     if(_calculated) {
-      sb.append("calculated", _calculated);
+      sb.append("calculated", _calculated)
+        .append("expression",
+                CustomToStringStyle.ignoreNull(getCalculationContext()));
     }
     if(_type.isTextual()) {
       sb.append("compressedUnicode", isCompressedUnicode())
@@ -1433,9 +1505,11 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
     if(_autoNumber) {
       sb.append("lastAutoNumber", _autoNumberGenerator.getLast());
     }
-    if(getComplexInfo() != null) {
-      sb.append("complexInfo", getComplexInfo());
-    }
+    sb.append("complexInfo", CustomToStringStyle.ignoreNull(getComplexInfo()))
+      .append("validator", CustomToStringStyle.ignoreNull(
+                  ((_validator != SimpleColumnValidator.INSTANCE) ?
+                   _validator : null)))
+      .append("defaultValue", CustomToStringStyle.ignoreNull(_defValue));
     return sb.toString();
   }
 
@@ -2334,6 +2408,11 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
       }
       return val;
     }
+
+    @Override
+    protected void appendToString(StringBuilder sb) {
+      sb.append("required=true");
+    }
   }
 
   /**
@@ -2359,5 +2438,10 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl> {
       }
       return valStr;
     }
+
+    @Override
+    protected void appendToString(StringBuilder sb) {
+      sb.append("allowZeroLength=false");
+    }
   }
 }

src/main/java/com/healthmarketscience/jackcess/impl/CustomToStringStyle.java

@@ -30,14 +30,15 @@ import org.apache.commons.lang.builder.ToStringBuilder;
  *
  * @author James Ahlborn
  */
-public class CustomToStringStyle extends StandardToStringStyle 
+public class CustomToStringStyle extends StandardToStringStyle
 {
   private static final long serialVersionUID = 0L;
 
   private static final String ML_FIELD_SEP = SystemUtils.LINE_SEPARATOR + "  ";
   private static final String IMPL_SUFFIX = "Impl";
   private static final int MAX_BYTE_DETAIL_LEN = 20;
-  
+  private static final Object IGNORE_ME = new Object();
+
   public static final CustomToStringStyle INSTANCE = new CustomToStringStyle() {
     private static final long serialVersionUID = 0L;
     {
@@ -59,7 +60,7 @@ public class CustomToStringStyle extends StandardToStringStyle
     }
   };
 
-  private CustomToStringStyle() {    
+  private CustomToStringStyle() {
   }
 
   public static ToStringBuilder builder(Object obj) {
@@ -70,6 +71,15 @@ public class CustomToStringStyle extends StandardToStringStyle
     return new ToStringBuilder(obj, VALUE_INSTANCE);
   }
 
+  @Override
+  public void append(StringBuffer buffer, String fieldName, Object value,
+                     Boolean fullDetail) {
+    if(value == IGNORE_ME) {
+      return;
+    }
+    super.append(buffer, fieldName, value, fullDetail);
+  }
+
   @Override
   protected void appendClassName(StringBuffer buffer, Object obj) {
     if(obj instanceof String) {
@@ -84,7 +94,7 @@ public class CustomToStringStyle extends StandardToStringStyle
   protected String getShortClassName(Class clss) {
     String shortName = super.getShortClassName(clss);
     if(shortName.endsWith(IMPL_SUFFIX)) {
-      shortName = shortName.substring(0, 
+      shortName = shortName.substring(0,
                                       shortName.length() - IMPL_SUFFIX.length());
     }
     int idx = shortName.lastIndexOf('.');
@@ -95,7 +105,7 @@ public class CustomToStringStyle extends StandardToStringStyle
   }
 
   @Override
-  protected void appendDetail(StringBuffer buffer, String fieldName, 
+  protected void appendDetail(StringBuffer buffer, String fieldName,
                               Object value) {
     if(value instanceof ByteBuffer) {
       appendDetail(buffer, (ByteBuffer)value);
@@ -105,7 +115,7 @@ public class CustomToStringStyle extends StandardToStringStyle
   }
 
   @Override
-  protected void appendDetail(StringBuffer buffer, String fieldName, 
+  protected void appendDetail(StringBuffer buffer, String fieldName,
                               Collection value) {
     buffer.append("[");
 
@@ -167,32 +177,36 @@ public class CustomToStringStyle extends StandardToStringStyle
   }
 
   @Override
-  protected void appendDetail(StringBuffer buffer, String fieldName, 
+  protected void appendDetail(StringBuffer buffer, String fieldName,
                               byte[] array) {
     appendDetail(buffer, PageChannel.wrap(array));
   }
 
-  private void appendValueDetail(StringBuffer buffer, String fieldName, 
+  private void appendValueDetail(StringBuffer buffer, String fieldName,
                                  Object value) {
     if (value == null) {
       appendNullText(buffer, fieldName);
     } else {
       appendInternal(buffer, fieldName, value, true);
-    }    
+    }
   }
 
   private static void appendDetail(StringBuffer buffer, ByteBuffer bb) {
     int len = bb.remaining();
     buffer.append("(").append(len).append(") ");
-    buffer.append(ByteUtil.toHexString(bb, bb.position(), 
+    buffer.append(ByteUtil.toHexString(bb, bb.position(),
                                        Math.min(len, MAX_BYTE_DETAIL_LEN)));
     if(len > MAX_BYTE_DETAIL_LEN) {
       buffer.append(" ...");
-    }      
+    }
   }
 
   private static String indent(Object obj) {
     return ((obj != null) ? obj.toString().replaceAll(
                 SystemUtils.LINE_SEPARATOR, ML_FIELD_SEP) : null);
   }
+
+  public static Object ignoreNull(Object obj) {
+    return ((obj != null) ? obj : IGNORE_ME);
+  }
 }

src/main/java/com/healthmarketscience/jackcess/impl/DBEvalContext.java

@@ -0,0 +1,88 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.text.SimpleDateFormat;
+import java.util.Map;
+
+import com.healthmarketscience.jackcess.expr.EvalConfig;
+import com.healthmarketscience.jackcess.expr.Function;
+import com.healthmarketscience.jackcess.expr.TemporalConfig;
+import com.healthmarketscience.jackcess.impl.expr.DefaultFunctions;
+import com.healthmarketscience.jackcess.impl.expr.Expressionator;
+import com.healthmarketscience.jackcess.impl.expr.RandomContext;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class DBEvalContext implements Expressionator.ParseContext, EvalConfig
+{
+  private static final int MAX_CACHE_SIZE = 10;
+
+  private final DatabaseImpl _db;
+  private Map<String,SimpleDateFormat> _sdfs;
+  private TemporalConfig _temporal;
+  private final RandomContext _rndCtx = new RandomContext();
+
+  public DBEvalContext(DatabaseImpl db) 
+  {
+    _db = db;
+  }
+
+  protected DatabaseImpl getDatabase() {
+    return _db;
+  }
+
+  public TemporalConfig getTemporalConfig() {
+    return _temporal;
+  }
+
+  public void setTemporalConfig(TemporalConfig temporal) {
+    _temporal = temporal;
+  }
+
+  public void putCustomExpressionFunction(Function func) {
+    // FIXME writeme
+  }
+
+  public Function getCustomExpressionFunction(String name) {
+    // FIXME writeme
+    return null;
+  }
+
+  public SimpleDateFormat createDateFormat(String formatStr) {
+    if(_sdfs == null) {
+      _sdfs = new SimpleCache<String,SimpleDateFormat>(MAX_CACHE_SIZE);
+    }
+    SimpleDateFormat sdf = _sdfs.get(formatStr);
+    if(formatStr == null) {
+      sdf = _db.createDateFormat(formatStr);
+      _sdfs.put(formatStr, sdf);
+    }
+    return sdf;
+  }
+
+  public float getRandom(Integer seed) {
+    return _rndCtx.getRandom(seed);
+  }
+
+  public Function getExpressionFunction(String name) {
+    // FIXME, support custom function context?
+    return DefaultFunctions.getFunction(name);
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/DatabaseImpl.java

@@ -63,6 +63,7 @@ import com.healthmarketscience.jackcess.RuntimeIOException;
 import com.healthmarketscience.jackcess.Table;
 import com.healthmarketscience.jackcess.TableBuilder;
 import com.healthmarketscience.jackcess.TableMetaData;
+import com.healthmarketscience.jackcess.expr.EvalConfig;
 import com.healthmarketscience.jackcess.impl.query.QueryImpl;
 import com.healthmarketscience.jackcess.query.Query;
 import com.healthmarketscience.jackcess.util.CaseInsensitiveColumnMatcher;
@@ -306,6 +307,8 @@ public class DatabaseImpl implements Database
   private boolean _enforceForeignKeys;
   /** whether or not auto numbers can be directly inserted by the user */
   private boolean _allowAutoNumInsert;
+  /** whether or not to evaluate expressions */
+  private boolean _evaluateExpressions;
   /** factory for ColumnValidators */
   private ColumnValidatorFactory _validatorFactory = SimpleColumnValidatorFactory.INSTANCE;
   /** cache of in-use tables */
@@ -331,6 +334,8 @@ public class DatabaseImpl implements Database
     FKEnforcer.initSharedState();
   /** Calendar for use interpreting dates/times in Columns */
   private Calendar _calendar;
+  /** shared context for evaluating expressions */
+  private DBEvalContext _evalCtx;
 
   /**
    * Open an existing Database.  If the existing file is not writeable or the
@@ -514,6 +519,7 @@ public class DatabaseImpl implements Database
     _columnOrder = getDefaultColumnOrder();
     _enforceForeignKeys = getDefaultEnforceForeignKeys();
     _allowAutoNumInsert = getDefaultAllowAutoNumberInsert();
+    _evaluateExpressions = getDefaultEvaluateExpressions();
     _fileFormat = fileFormat;
     _pageChannel = new PageChannel(channel, closeChannel, _format, autoSync);
     _timeZone = ((timeZone == null) ? getDefaultTimeZone() : timeZone);
@@ -686,6 +692,16 @@ public class DatabaseImpl implements Database
     _allowAutoNumInsert = allowAutoNumInsert;
   }
 
+  public boolean isEvaluateExpressions() {
+    return _evaluateExpressions;
+  }
+
+  public void setEvaluateExpressions(Boolean evaluateExpressions) {
+    if(evaluateExpressions == null) {
+      evaluateExpressions = getDefaultEvaluateExpressions();
+    }
+    _evaluateExpressions = evaluateExpressions;
+  }
 
   public ColumnValidatorFactory getColumnValidatorFactory() {
     return _validatorFactory;
@@ -716,6 +732,20 @@ public class DatabaseImpl implements Database
     return _calendar;
   }
 
+  public EvalConfig getEvalConfig() {
+    return getEvalContext();
+  }
+
+  /**
+   * @usage _advanced_method_
+   */
+  DBEvalContext getEvalContext() {
+    if(_evalCtx == null) {
+      _evalCtx = new DBEvalContext(this);
+    }
+    return _evalCtx;
+  }
+
   /**
    * Returns a SimpleDateFormat for the given format string which is
    * configured with a compatible Calendar instance (see
@@ -1796,6 +1826,18 @@ public class DatabaseImpl implements Database
     return((name == null) || (name.trim().length() == 0));
   }
 
+  /**
+   * Returns the given string trimmed, or {@code null} if the string is {@code
+   * null} or empty.
+   */
+  public static String trimToNull(String str) {
+    if(str == null) {
+      return null;
+    }
+    str = str.trim();
+    return((str.length() > 0) ? str : null);
+  }
+
   @Override
   public String toString() {
     return ToStringBuilder.reflectionToString(this);
@@ -1958,6 +2000,21 @@ public class DatabaseImpl implements Database
     return false;
   }
 
+  /**
+   * Returns the default enable expression evaluation policy.  This defaults to
+   * {@code false}, but can be overridden using the system
+   * property {@value com.healthmarketscience.jackcess.Database#ENABLE_EXPRESSION_EVALUATION_PROPERTY}.
+   * @usage _advanced_method_
+   */
+  public static boolean getDefaultEvaluateExpressions()
+  {
+    String prop = System.getProperty(ENABLE_EXPRESSION_EVALUATION_PROPERTY);
+    if(prop != null) {
+      return Boolean.TRUE.toString().equalsIgnoreCase(prop);
+    }
+    return false;
+  }
+
   /**
    * Copies the given db InputStream to the given channel using the most
    * efficient means possible.

src/main/java/com/healthmarketscience/jackcess/impl/InternalColumnValidator.java

@@ -20,6 +20,7 @@ import java.io.IOException;
 
 import com.healthmarketscience.jackcess.Column;
 import com.healthmarketscience.jackcess.util.ColumnValidator;
+import com.healthmarketscience.jackcess.util.SimpleColumnValidator;
 
 /**
  * Base class for ColumnValidator instances handling "internal" validation
@@ -31,7 +32,7 @@ abstract class InternalColumnValidator implements ColumnValidator
 {
   private ColumnValidator _delegate;
 
-  protected InternalColumnValidator(ColumnValidator delegate) 
+  protected InternalColumnValidator(ColumnValidator delegate)
   {
     _delegate = delegate;
   }
@@ -57,6 +58,24 @@ abstract class InternalColumnValidator implements ColumnValidator
     return internalValidate(col, val);
   }
 
-  protected abstract Object internalValidate(Column col, Object val) 
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder().append("{");
+    if(_delegate instanceof InternalColumnValidator) {
+      ((InternalColumnValidator)_delegate).appendToString(sb);
+    } else if(_delegate != SimpleColumnValidator.INSTANCE) {
+      sb.append("custom=").append(_delegate);
+    }
+    if(sb.length() > 1) {
+      sb.append(";");
+    }
+    appendToString(sb);
+    sb.append("}");
+    return sb.toString();
+  }
+
+  protected abstract void appendToString(StringBuilder sb);
+
+  protected abstract Object internalValidate(Column col, Object val)
     throws IOException;
 }

src/main/java/com/healthmarketscience/jackcess/impl/PropertyMaps.java

@@ -125,6 +125,12 @@ public class PropertyMaps implements Iterable<PropertyMapImpl>
       .toString();
   }
 
+  public static String getTrimmedStringProperty(
+      PropertyMap props, String propName)
+  {
+    return DatabaseImpl.trimToNull((String)props.getValue(propName));
+  }
+
   /**
    * Utility class for reading/writing property blocks.
    */

src/main/java/com/healthmarketscience/jackcess/impl/RowEvalContext.java

@@ -0,0 +1,64 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import com.healthmarketscience.jackcess.expr.EvalException;
+import com.healthmarketscience.jackcess.expr.Identifier;
+import com.healthmarketscience.jackcess.expr.Value;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public abstract class RowEvalContext extends BaseEvalContext
+{
+  private Object[] _row;
+
+  public RowEvalContext(DatabaseImpl db) {
+    super(db.getEvalContext());
+  }
+
+  protected void setRow(Object[] row) {
+    _row = row;
+  }
+
+  protected void reset() {
+    _row = null;
+  }
+
+  @Override
+  public Value getIdentifierValue(Identifier identifier) {
+
+    TableImpl table = getTable();
+
+    // we only support getting column values in this table from the current
+    // row
+    if(!table.isThisTable(identifier) ||
+       (identifier.getPropertyName() != null)) {
+      throw new EvalException("Cannot access fields outside this table for " +
+                              identifier);
+    }
+
+    ColumnImpl col = table.getColumn(identifier.getObjectName());
+
+    Object val = col.getRowValue(_row);
+
+    return toValue(val, col.getType());
+  }
+
+  protected abstract TableImpl getTable();
+}

src/main/java/com/healthmarketscience/jackcess/impl/RowValidatorEvalContext.java

@@ -0,0 +1,67 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.io.IOException;
+
+import com.healthmarketscience.jackcess.InvalidValueException;
+import com.healthmarketscience.jackcess.impl.expr.Expressionator;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class RowValidatorEvalContext extends RowEvalContext
+{
+  private final TableImpl _table;
+  private String _helpStr;
+
+  public RowValidatorEvalContext(TableImpl table) {
+    super(table.getDatabase());
+    _table = table;
+  }
+
+  RowValidatorEvalContext setExpr(String exprStr, String helpStr) {
+    setExpr(Expressionator.Type.RECORD_VALIDATOR, exprStr);
+    _helpStr = helpStr;
+    return this;
+  }
+
+  @Override
+  protected TableImpl getTable() {
+    return _table;
+  }
+
+  public void validate(Object[] row) throws IOException {
+    try {
+      setRow(row);
+      Boolean result = (Boolean)eval();
+      // FIXME how to handle null?
+      if(!result) {
+        String msg = ((_helpStr != null) ? _helpStr : "Invalid row");
+        throw new InvalidValueException(withErrorContext(msg));
+      }
+    } finally {
+      reset();
+    }
+  }
+
+  @Override
+  protected String withErrorContext(String msg) {
+    return _table.withErrorContext(msg);
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/TableImpl.java

@@ -49,8 +49,10 @@ import com.healthmarketscience.jackcess.PropertyMap;
 import com.healthmarketscience.jackcess.Row;
 import com.healthmarketscience.jackcess.RowId;
 import com.healthmarketscience.jackcess.Table;
+import com.healthmarketscience.jackcess.expr.Identifier;
 import com.healthmarketscience.jackcess.util.ErrorHandler;
 import com.healthmarketscience.jackcess.util.ExportUtil;
+import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -134,6 +136,8 @@ public class TableImpl implements Table, PropertyMaps.Owner
   private final List<ColumnImpl> _varColumns = new ArrayList<ColumnImpl>();
   /** List of autonumber columns in this table, ordered by column number */
   private final List<ColumnImpl> _autoNumColumns = new ArrayList<ColumnImpl>(1);
+  /** handler for calculated columns */
+  private final CalcColEvaluator _calcColEval = new CalcColEvaluator();
   /** List of indexes on this table (multiple logical indexes may be backed by
       the same index data) */
   private final List<IndexImpl> _indexes = new ArrayList<IndexImpl>();
@@ -179,6 +183,8 @@ public class TableImpl implements Table, PropertyMaps.Owner
   private Boolean _allowAutoNumInsert;
   /** foreign-key enforcer for this table */
   private final FKEnforcer _fkEnforcer;
+  /** table validator if any (and enabled) */
+  private RowValidatorEvalContext _rowValidator;
 
   /** default cursor for iterating through the table, kept here for basic
       table traversal */
@@ -281,11 +287,36 @@ public class TableImpl implements Table, PropertyMaps.Owner
     _fkEnforcer = new FKEnforcer(this);
 
     if(!isSystem()) {
-      // after fully constructed, allow column validator to be configured (but
-      // only for user tables)
+      // after fully constructed, allow column/row validators to be configured
+      // (but only for user tables)
       for(ColumnImpl col : _columns) {
         col.initColumnValidator();
       }
+
+      reloadRowValidator();
+    }
+  }
+
+  private void reloadRowValidator() throws IOException {
+
+    // reset table row validator before proceeding
+    _rowValidator = null;
+
+    if(!getDatabase().isEvaluateExpressions()) {
+      return;
+    }
+
+    PropertyMap props = getProperties();
+
+    String exprStr = PropertyMaps.getTrimmedStringProperty(
+        props, PropertyMap.VALIDATION_RULE_PROP);
+
+    if(exprStr != null) {
+      String helpStr = PropertyMaps.getTrimmedStringProperty(
+          props, PropertyMap.VALIDATION_TEXT_PROP);
+
+      _rowValidator = new RowValidatorEvalContext(this)
+        .setExpr(exprStr, helpStr);
     }
   }
 
@@ -444,9 +475,16 @@ public class TableImpl implements Table, PropertyMaps.Owner
   }
 
   public void propertiesUpdated() throws IOException {
+    // propagate update to columns
     for(ColumnImpl col : _columns) {
       col.propertiesUpdated();
     }
+
+    reloadRowValidator();
+
+    // calculated columns will need to be re-sorted (their expressions may
+    // have changed when their properties were updated)
+    _calcColEval.reSort();
   }
 
   public List<IndexImpl> getIndexes() {
@@ -1290,6 +1328,9 @@ public class TableImpl implements Table, PropertyMaps.Owner
     if(newCol.isAutoNumber()) {
       _autoNumColumns.add(newCol);
     }
+    if(newCol.isCalculated()) {
+      _calcColEval.add(newCol);
+    }
 
     if(umapPos >= 0) {
       // read column usage map
@@ -1925,6 +1966,7 @@ public class TableImpl implements Table, PropertyMaps.Owner
 
     Collections.sort(_columns);
     initAutoNumberColumns();
+    initCalculatedColumns();
 
     // setup the data index for the columns
     int colIdx = 0;
@@ -2187,8 +2229,12 @@ public class TableImpl implements Table, PropertyMaps.Owner
           // handle various value massaging activities
           for(ColumnImpl column : _columns) {
             if(!column.isAutoNumber()) {
+              Object val = column.getRowValue(row);
+              if(val == null) {
+                val = column.generateDefaultValue();
+              }
               // pass input value through column validator
-              column.setRowValue(row, column.validate(column.getRowValue(row)));
+              column.setRowValue(row, column.validate(val));
             }
           }
 
@@ -2196,6 +2242,15 @@ public class TableImpl implements Table, PropertyMaps.Owner
           handleAutoNumbersForAdd(row, writeRowState);
           ++autoNumAssignCount;
 
+          // need to assign calculated values after all the other fields are
+          // filled in but before final validation
+          _calcColEval.calculate(row);
+
+          // run row validation if enabled
+          if(_rowValidator != null) {
+            _rowValidator.validate(row);
+          }
+
           // write the row of data to a temporary buffer
           ByteBuffer rowData = createRow(
               row, _writeRowBufferH.getPageBuffer(getPageChannel()));
@@ -2440,6 +2495,15 @@ public class TableImpl implements Table, PropertyMaps.Owner
       // fill in autonumbers
       handleAutoNumbersForUpdate(row, rowBuffer, rowState);
 
+      // need to assign calculated values after all the other fields are
+      // filled in but before final validation
+      _calcColEval.calculate(row);
+
+      // run row validation if enabled
+      if(_rowValidator != null) {
+        _rowValidator.validate(row);
+      }
+
       // generate new row bytes
       ByteBuffer newRowData = createRow(
           row, _writeRowBufferH.getPageBuffer(getPageChannel()), oldRowSize,
@@ -2661,6 +2725,7 @@ public class TableImpl implements Table, PropertyMaps.Owner
     return dataPage;
   }
 
+  // exposed for unit tests
   protected ByteBuffer createRow(Object[] rowArray, ByteBuffer buffer)
     throws IOException
   {
@@ -2984,6 +3049,7 @@ public class TableImpl implements Table, PropertyMaps.Owner
       .append("columnCount", _columns.size())
       .append("indexCount(data)", _indexCount)
       .append("logicalIndexCount", _logicalIndexCount)
+      .append("validator", CustomToStringStyle.ignoreNull(_rowValidator))
       .append("columns", _columns)
       .append("indexes", _indexes)
       .append("ownedPages", _ownedPages)
@@ -3164,6 +3230,20 @@ public class TableImpl implements Table, PropertyMaps.Owner
     }
   }
 
+  private void initCalculatedColumns() {
+    for(ColumnImpl c : _columns) {
+      if(c.isCalculated()) {
+        _calcColEval.add(c);
+      }
+    }
+  }
+
+  boolean isThisTable(Identifier identifier) {
+    String collectionName = identifier.getCollectionName();
+    return ((collectionName == null) ||
+            collectionName.equalsIgnoreCase(getName()));
+  }
+
   /**
    * Returns {@code true} if a row of the given size will fit on the given
    * data page, {@code false} otherwise.
@@ -3190,7 +3270,7 @@ public class TableImpl implements Table, PropertyMaps.Owner
     return copy;
   }
 
-  private String withErrorContext(String msg) {
+  String withErrorContext(String msg) {
     return withErrorContext(msg, getDatabase(), getName());
   }
 
@@ -3493,4 +3573,73 @@ public class TableImpl implements Table, PropertyMaps.Owner
     }
   }
 
+  /**
+   * Utility for managing calculated columns.  Calculated columns need to be
+   * evaluated in dependency order.
+   */
+  private class CalcColEvaluator
+  {
+    /** List of calculated columns in this table, ordered by calculation
+        dependency */
+    private final List<ColumnImpl> _calcColumns = new ArrayList<ColumnImpl>(1);
+    private boolean _sorted;
+
+    public void add(ColumnImpl col) {
+      if(!getDatabase().isEvaluateExpressions()) {
+        return;
+      }
+      _calcColumns.add(col);
+      // whenever we add new columns, we need to re-sort
+      _sorted = false;
+    }
+
+    public void reSort() {
+      // mark columns for re-sort on next use
+      _sorted = false;
+    }
+
+    public void calculate(Object[] row) throws IOException {
+      if(!_sorted) {
+        sortColumnsByDeps();
+        _sorted = true;
+      }
+
+      for(ColumnImpl col : _calcColumns) {
+        Object rowValue = col.getCalculationContext().eval(row);
+        col.setRowValue(row, rowValue);
+      }
+    }
+
+    private void sortColumnsByDeps() {
+
+      // a topological sort sorts nodes where A -> B such that A ends up in
+      // the list before B (assuming that we are working with a DAG).  In our
+      // case, we return "descendent" info as Field1 -> Field2 (where Field1
+      // uses Field2 in its calculation).  This means that in order to
+      // correctly calculate Field1, we need to calculate Field2 first, and
+      // hence essentially need the reverse topo sort (a list where Field2
+      // comes before Field1).
+      (new TopoSorter<ColumnImpl>(_calcColumns, TopoSorter.REVERSE) {
+        @Override
+        protected void getDescendents(ColumnImpl from,
+                                      List<ColumnImpl> descendents) {
+
+          Set<Identifier> identifiers = new LinkedHashSet<Identifier>();
+          from.getCalculationContext().collectIdentifiers(identifiers);
+
+          for(Identifier identifier : identifiers) {
+            if(isThisTable(identifier)) {
+              String colName = identifier.getObjectName();
+              for(ColumnImpl calcCol : _calcColumns) {
+                // we only care if the identifier is another calc field
+                if(calcCol.getName().equalsIgnoreCase(colName)) {
+                  descendents.add(calcCol);
+                }
+              }
+            }
+          }
+        }
+      }).sort();
+    }
+  }
 }

src/main/java/com/healthmarketscience/jackcess/impl/TopoSorter.java

@@ -0,0 +1,114 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public abstract class TopoSorter<E>
+{
+  public static final boolean REVERSE = true;
+
+  // https://en.wikipedia.org/wiki/Topological_sorting
+  private static final int UNMARKED = 0;
+  private static final int TEMP_MARK = 1;
+  private final static int PERM_MARK = 2;
+
+  private final List<E> _values;
+  private final List<Node<E>> _nodes = new ArrayList<Node<E>>();
+  private final boolean _reverse;
+
+  protected TopoSorter(List<E> values, boolean reverse) {
+    _values = values;
+    _reverse = reverse;
+  }
+
+  public void sort() {
+      
+    for(E val : _values) {
+      Node<E> node = new Node<E>(val);
+      getDescendents(val, node._descs);
+
+      // build the internal list in reverse so that we maintain the "original"
+      // order of items which we don't need to re-arrange
+      _nodes.add(0, node);
+    }
+
+    _values.clear();
+
+    for(Node<E> node : _nodes) {
+      if(node._mark != UNMARKED) {
+        continue;
+      }
+
+      visit(node);
+    }
+  }
+
+  private void visit(Node<E> node) {
+    
+    if(node._mark == PERM_MARK) {
+      return;
+    }
+
+    if(node._mark == TEMP_MARK) {
+      throw new IllegalStateException("Cycle detected");
+    }
+
+    node._mark = TEMP_MARK;
+
+    for(E descVal : node._descs) {
+      Node<E> desc = findDescendent(descVal);
+      visit(desc);
+    }
+
+    node._mark = PERM_MARK;
+
+    if(_reverse) {
+      _values.add(node._val);
+    } else {
+      _values.add(0, node._val);
+    }
+  }
+
+  private Node<E> findDescendent(E val) {
+    for(Node<E> node : _nodes) {
+      if(node._val == val) {
+        return node;
+      }
+    }
+    throw new IllegalStateException("Unknown descendent " + val);
+  }
+
+  protected abstract void getDescendents(E from, List<E> descendents);
+
+  
+  private static class Node<E>
+  {
+    private final E _val;
+    private final List<E> _descs = new ArrayList<E>();
+    private int _mark = UNMARKED;
+
+    private Node(E val) {
+      _val = val;
+    }
+  }
+}

src/main/java/com/healthmarketscience/jackcess/impl/expr/BuiltinOperators.java

@@ -32,13 +32,13 @@ import com.healthmarketscience.jackcess.impl.ColumnImpl;
  *
  * @author James Ahlborn
  */
-public class BuiltinOperators 
+public class BuiltinOperators
 {
   private static final String DIV_BY_ZERO = "/ by zero";
 
   private static final double MIN_INT = Integer.MIN_VALUE;
   private static final double MAX_INT = Integer.MAX_VALUE;
-  
+
   public static final Value NULL_VAL = new BaseValue() {
     @Override public boolean isNull() {
       return true;
@@ -58,7 +58,7 @@ public class BuiltinOperators
   public static final Value ZERO_VAL = FALSE_VAL;
 
   public static final RoundingMode ROUND_MODE = RoundingMode.HALF_EVEN;
-  
+
   private enum CoercionType {
     SIMPLE(true, true), GENERAL(false, true), COMPARE(false, false);
 
@@ -118,11 +118,11 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.SIMPLE);
 
     switch(mathType) {
-    case STRING: 
+    case STRING:
       // string '+' is a null-propagation (handled above) concat
       return nonNullConcat(param1, param2);
     case DATE:
@@ -148,7 +148,7 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.SIMPLE);
 
     switch(mathType) {
@@ -176,7 +176,7 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.GENERAL);
 
     switch(mathType) {
@@ -201,7 +201,7 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.GENERAL);
 
     switch(mathType) {
@@ -235,7 +235,7 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.GENERAL);
     if(mathType == Value.Type.STRING) {
       throw new EvalException("Unexpected type " + mathType);
@@ -249,7 +249,7 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.GENERAL);
 
     // jdk only supports general pow() as doubles, let's go with that
@@ -269,7 +269,7 @@ public class BuiltinOperators
       return NULL_VAL;
     }
 
-    Value.Type mathType = getMathTypePrecedence(param1, param2, 
+    Value.Type mathType = getMathTypePrecedence(param1, param2,
                                                 CoercionType.GENERAL);
 
     if(mathType == Value.Type.STRING) {
@@ -301,7 +301,7 @@ public class BuiltinOperators
       // null propagation
       return NULL_VAL;
     }
-    
+
     return toValue(!param1.getAsBoolean());
   }
 
@@ -462,7 +462,7 @@ public class BuiltinOperators
       // null propagation
       return NULL_VAL;
     }
-    
+
     return toValue(pattern.matcher(param1.getAsString()).matches());
   }
 
@@ -515,7 +515,7 @@ public class BuiltinOperators
     return not(in(param1, params));
   }
 
-  
+
   private static boolean anyParamIsNull(Value param1, Value param2) {
     return (param1.isNull() || param2.isNull());
   }
@@ -529,7 +529,7 @@ public class BuiltinOperators
       Value param1, Value param2)
   {
     // note that comparison does not do string to num coercion
-    Value.Type compareType = getMathTypePrecedence(param1, param2, 
+    Value.Type compareType = getMathTypePrecedence(param1, param2,
                                                    CoercionType.COMPARE);
 
     switch(compareType) {
@@ -589,7 +589,11 @@ public class BuiltinOperators
     return toValue(type, new Date(ColumnImpl.fromDateDouble(dd, fmt.getCalendar())),
                    fmt);
   }
-  
+
+  public static Value toValue(EvalContext ctx, Value.Type type, Date d) {
+    return toValue(type, d, getDateFormatForType(ctx, type));
+  }
+
   public static Value toValue(Value.Type type, Date d, DateFormat fmt) {
     switch(type) {
     case DATE:
@@ -602,8 +606,8 @@ public class BuiltinOperators
       throw new EvalException("Unexpected date/time type " + type);
     }
   }
-  
-  static Value toDateValue(EvalContext ctx, Value.Type type, double v, 
+
+  static Value toDateValue(EvalContext ctx, Value.Type type, double v,
                            Value param1, Value param2)
   {
     DateFormat fmt = null;
@@ -675,15 +679,18 @@ public class BuiltinOperators
     if(cType._preferTemporal &&
        (t1.isTemporal() || t2.isTemporal())) {
       return (t1.isTemporal() ?
-              (t2.isTemporal() ? 
+              (t2.isTemporal() ?
                // for mixed temporal types, always go to date/time
                Value.Type.DATE_TIME : t1) :
               t2);
     }
 
-    t1 = t1.getPreferredNumericType();
-    t2 = t2.getPreferredNumericType();
+    return getPreferredNumericType(t1.getPreferredNumericType(),
+                                   t2.getPreferredNumericType());
+  }
 
+  private static Value.Type getPreferredNumericType(Value.Type t1, Value.Type t2)
+  {
     // if both types are integral, choose "largest"
     if(t1.isIntegral() && t2.isIntegral()) {
       return max(t1, t2);
@@ -719,7 +726,14 @@ public class BuiltinOperators
 
     try {
       // see if string can be coerced to a number
-      strParam.getAsBigDecimal();
+      BigDecimal num = strParam.getAsBigDecimal();
+      if(prefType.isNumeric()) {
+        // re-evaluate the numeric type choice based on the type of the parsed
+        // number
+        Value.Type numType = ((num.stripTrailingZeros().scale() > 0) ?
+                              Value.Type.BIG_DEC : Value.Type.LONG);
+        prefType = getPreferredNumericType(numType, prefType);
+      }
       return prefType;
     } catch(NumberFormatException ignored) {
       // not a number

src/main/java/com/healthmarketscience/jackcess/impl/expr/ExpressionTokenizer.java

@@ -44,6 +44,7 @@ class ExpressionTokenizer
 {
   private static final int EOF = -1;
   private static final char QUOTED_STR_CHAR = '"';
+  private static final char SINGLE_QUOTED_STR_CHAR = '\'';
   private static final char OBJ_NAME_START_CHAR = '[';
   private static final char OBJ_NAME_END_CHAR = ']';
   private static final char DATE_LIT_QUOTE_CHAR = '#';
@@ -75,7 +76,7 @@ class ExpressionTokenizer
     setCharFlag(IS_COMP_FLAG, '<', '>', '=');
     setCharFlag(IS_DELIM_FLAG, '.', '!', ',', '(', ')');
     setCharFlag(IS_SPACE_FLAG, ' ', '\n', '\r', '\t');
-    setCharFlag(IS_QUOTE_FLAG, '"', '#', '[', ']');
+    setCharFlag(IS_QUOTE_FLAG, '"', '#', '[', ']', '\'');
   }
 
   /**
@@ -142,11 +143,12 @@ class ExpressionTokenizer
 
           switch(c) {
           case QUOTED_STR_CHAR:
-            tokens.add(new Token(TokenType.LITERAL, null, parseQuotedString(buf),
-                                 Value.Type.STRING));
+          case SINGLE_QUOTED_STR_CHAR:
+            tokens.add(new Token(TokenType.LITERAL, null, 
+                                 parseQuotedString(buf, c), Value.Type.STRING));
             break;
           case DATE_LIT_QUOTE_CHAR:
-            tokens.add(parseDateLiteralString(buf));
+            tokens.add(parseDateLiteral(buf));
             break;
           case OBJ_NAME_START_CHAR:
             tokens.add(new Token(TokenType.OBJ_NAME, parseObjNameString(buf)));
@@ -237,40 +239,21 @@ class ExpressionTokenizer
     return sb.toString();
   }
 
-  private static String parseQuotedString(ExprBuf buf) {
-    StringBuilder sb = buf.getScratchBuffer();
-
-    boolean complete = false;
-    while(buf.hasNext()) {
-      char c = buf.next();
-      if(c == QUOTED_STR_CHAR) {
-        int nc = buf.peekNext();
-        if(nc == QUOTED_STR_CHAR) {
-          sb.append(QUOTED_STR_CHAR);
-          buf.next();
-        } else {
-          complete = true;
-          break;
-        }
-      }
-
-      sb.append(c);
-    }
-
-    if(!complete) {
-      throw new ParseException("Missing closing '" + QUOTED_STR_CHAR + 
-                               "' for quoted string " + buf);
-    }
-
-    return sb.toString();
+  private static String parseQuotedString(ExprBuf buf, char quoteChar) {
+    return parseStringUntil(buf, quoteChar, null, true);
   }
 
   private static String parseObjNameString(ExprBuf buf) {
-    return parseStringUntil(buf, OBJ_NAME_END_CHAR, OBJ_NAME_START_CHAR);
+    return parseStringUntil(buf, OBJ_NAME_END_CHAR, OBJ_NAME_START_CHAR, false);
+  }
+
+  private static String parseDateLiteralString(ExprBuf buf) {
+    return parseStringUntil(buf, DATE_LIT_QUOTE_CHAR, null, false);
   }
 
   private static String parseStringUntil(ExprBuf buf, char endChar, 
-                                         Character startChar) 
+                                         Character startChar,
+                                         boolean allowDoubledEscape) 
   {
     StringBuilder sb = buf.getScratchBuffer();
 
@@ -278,8 +261,13 @@ class ExpressionTokenizer
     while(buf.hasNext()) {
       char c = buf.next();
       if(c == endChar) {
-        complete = true;
-        break;
+        if(allowDoubledEscape && (buf.peekNext() == endChar)) {
+          sb.append(endChar);
+          buf.next();
+        } else {
+          complete = true;
+          break;
+        }
       } else if((startChar != null) &&
                 (startChar == c)) {
         throw new ParseException("Missing closing '" + endChar +
@@ -297,10 +285,10 @@ class ExpressionTokenizer
     return sb.toString();
   }
 
-  private static Token parseDateLiteralString(ExprBuf buf) 
+  private static Token parseDateLiteral(ExprBuf buf) 
   {
     TemporalConfig cfg = buf.getTemporalConfig();
-    String dateStr = parseStringUntil(buf, DATE_LIT_QUOTE_CHAR, null);
+    String dateStr = parseDateLiteralString(buf);
     
     boolean hasDate = (dateStr.indexOf(cfg.getDateSeparator()) >= 0);
     boolean hasTime = (dateStr.indexOf(cfg.getTimeSeparator()) >= 0);

src/main/java/com/healthmarketscience/jackcess/impl/expr/Expressionator.java

@@ -21,6 +21,7 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.Deque;
@@ -38,8 +39,9 @@ import com.healthmarketscience.jackcess.DatabaseBuilder;
 import com.healthmarketscience.jackcess.expr.EvalContext;
 import com.healthmarketscience.jackcess.expr.Expression;
 import com.healthmarketscience.jackcess.expr.Function;
-import com.healthmarketscience.jackcess.expr.TemporalConfig;
+import com.healthmarketscience.jackcess.expr.Identifier;
 import com.healthmarketscience.jackcess.expr.ParseException;
+import com.healthmarketscience.jackcess.expr.TemporalConfig;
 import com.healthmarketscience.jackcess.expr.Value;
 import com.healthmarketscience.jackcess.impl.expr.ExpressionTokenizer.Token;
 import com.healthmarketscience.jackcess.impl.expr.ExpressionTokenizer.TokenType;
@@ -615,7 +617,9 @@ public class Expressionator
     // object identifiers can be formatted like:
     //     "[Collection name]![Object name].[Property name]"
     // However, in practice, they only ever seem to be (at most) two levels
-    // and only use '.'.
+    // and only use '.'.  Apparently '!' is actually a special late-bind
+    // operator (not sure it makes a difference for this code?), see:
+    // http://bytecomb.com/the-bang-exclamation-operator-in-vba/
     Deque<String> objNames = new LinkedList<String>();
     objNames.add(firstTok.getValueStr());
 
@@ -641,17 +645,21 @@ public class Expressionator
       break;
     }
 
-    if(atSep || (objNames.size() > 3)) {
+    int numNames = objNames.size();
+    if(atSep || (numNames > 3)) {
       throw new ParseException("Invalid object reference " + buf);
     }
 
     // names are in reverse order
-    String fieldName = objNames.poll();
+    String propName = null;
+    if(numNames == 3) {
+      propName = objNames.poll();
+    }
     String objName = objNames.poll();
     String collectionName = objNames.poll();
 
     buf.setPendingExpr(
-        new EObjValue(collectionName, objName, fieldName));
+        new EObjValue(new Identifier(collectionName, objName, propName)));
   }
   
   private static void parseDelimExpression(Token firstTok, TokBuf buf) {
@@ -1387,7 +1395,7 @@ public class Expressionator
     protected boolean isConditionalExpr() {
       return false;
     }
-    
+
     protected void toString(StringBuilder sb, boolean isDebug) {
       if(isDebug) {
         sb.append("<").append(getClass().getSimpleName()).append(">{");
@@ -1458,6 +1466,8 @@ public class Expressionator
 
     public abstract Value eval(EvalContext ctx);
 
+    public abstract void collectIdentifiers(Collection<Identifier> identifiers);
+    
     protected abstract void toExprString(StringBuilder sb, boolean isDebug);
   }
 
@@ -1481,6 +1491,11 @@ public class Expressionator
       return _val;
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      // none
+    }
+    
     @Override 
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       sb.append(_str);
@@ -1497,6 +1512,10 @@ public class Expressionator
     public Value eval(EvalContext ctx) {
       return ctx.getThisColumnValue();
     }
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      // none
+    }
     @Override 
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       sb.append("<THIS_COL>");
@@ -1522,6 +1541,11 @@ public class Expressionator
       return _val;
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      // none
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       if(_val.getType() == Value.Type.STRING) {
@@ -1536,15 +1560,10 @@ public class Expressionator
 
   private static final class EObjValue extends Expr
   {
-    private final String _collectionName;
-    private final String _objName;
-    private final String _fieldName;
-
+    private final Identifier _identifier;
 
-    private EObjValue(String collectionName, String objName, String fieldName) {
-      _collectionName = collectionName;
-      _objName = objName;
-      _fieldName = fieldName;
+    private EObjValue(Identifier identifier) {
+      _identifier = identifier;
     }
 
     @Override
@@ -1554,18 +1573,17 @@ public class Expressionator
 
     @Override
     public Value eval(EvalContext ctx) {
-      return ctx.getRowValue(_collectionName, _objName, _fieldName);
+      return ctx.getIdentifierValue(_identifier);
+    }
+
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      identifiers.add(_identifier);
     }
 
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
-      if(_collectionName != null) {
-        sb.append("[").append(_collectionName).append("].");
-      }
-      if(_objName != null) {
-        sb.append("[").append(_objName).append("].");
-      }
-      sb.append("[").append(_fieldName).append("]");
+      sb.append(_identifier);
     }
   }
 
@@ -1592,6 +1610,11 @@ public class Expressionator
       return _expr.eval(ctx);
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      _expr.collectIdentifiers(identifiers);
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       sb.append("(");
@@ -1620,6 +1643,13 @@ public class Expressionator
       return _func.eval(ctx, exprListToValues(_params, ctx));
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      for(Expr param : _params) {
+        param.collectIdentifiers(identifiers);
+      }
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       sb.append(_func.getName()).append("(");
@@ -1670,6 +1700,12 @@ public class Expressionator
       _right = right;
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      _left.collectIdentifiers(identifiers);
+      _right.collectIdentifiers(identifiers);
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       _left.toString(sb, isDebug);
@@ -1723,6 +1759,11 @@ public class Expressionator
       return ((UnaryOp)_op).eval(ctx, _expr.eval(ctx));
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      _expr.collectIdentifiers(identifiers);
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       sb.append(_op);
@@ -1812,6 +1853,11 @@ public class Expressionator
       _expr = left;
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      _expr.collectIdentifiers(identifiers);
+    }
+
     @Override
     protected boolean isConditionalExpr() {
       return true;
@@ -1890,6 +1936,13 @@ public class Expressionator
                       exprListToDelayedValues(_exprs, ctx), null);
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      for(Expr expr : _exprs) {
+        expr.collectIdentifiers(identifiers);
+      }
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       _expr.toString(sb, isDebug);
@@ -1932,6 +1985,13 @@ public class Expressionator
                       new DelayedValue(_endRangeExpr, ctx));
     }
 
+    @Override
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      super.collectIdentifiers(identifiers);
+      _startRangeExpr.collectIdentifiers(identifiers);
+      _endRangeExpr.collectIdentifiers(identifiers);
+    }
+
     @Override
     protected void toExprString(StringBuilder sb, boolean isDebug) {
       _expr.toString(sb, isDebug);
@@ -1976,6 +2036,10 @@ public class Expressionator
       return _expr.isConstant();
     }
 
+    public void collectIdentifiers(Collection<Identifier> identifiers) {
+      _expr.collectIdentifiers(identifiers);
+    }
+
     @Override
     public String toString() {
       return _expr.toString();

src/main/java/com/healthmarketscience/jackcess/impl/expr/StringValue.java

@@ -29,7 +29,7 @@ public class StringValue extends BaseValue
   private final String _val;
   private Object _num;
 
-  public StringValue(String val) 
+  public StringValue(String val)
   {
     _val = val;
   }
@@ -79,9 +79,9 @@ public class StringValue extends BaseValue
         return (BigDecimal)_num;
       } catch(NumberFormatException nfe) {
         _num = NOT_A_NUMBER;
-        throw nfe;
+        // fall through to throw...
       }
     }
-    throw new NumberFormatException();
+    throw new NumberFormatException("Invalid number '" + _val + "'");
   }
 }

src/test/java/com/healthmarketscience/jackcess/impl/TopoSorterTest.java

@@ -0,0 +1,166 @@
+/*
+Copyright (c) 2018 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class TopoSorterTest extends TestCase
+{
+
+  public TopoSorterTest(String name) {
+    super(name);
+  }
+
+  public void testTopoSort() throws Exception
+  {
+    doTopoTest(Arrays.asList("A", "B", "C"),
+               Arrays.asList("A", "B", "C"));
+
+    doTopoTest(Arrays.asList("B", "A", "C"),
+               Arrays.asList("A", "B", "C"),
+               "B", "C",
+               "A", "B");
+
+    try {
+      doTopoTest(Arrays.asList("B", "A", "C"),
+                 Arrays.asList("C", "B", "A"),
+                 "B", "C",
+                 "A", "B",
+                 "C", "A");
+      fail("IllegalStateException should have been thrown");
+    } catch(IllegalStateException expected) {
+      // success
+      assertTrue(expected.getMessage().startsWith("Cycle"));
+    }
+
+    try {
+      doTopoTest(Arrays.asList("B", "A", "C"),
+                 Arrays.asList("C", "B", "A"),
+                 "B", "D");
+      fail("IllegalStateException should have been thrown");
+    } catch(IllegalStateException expected) {
+      // success
+      assertTrue(expected.getMessage().startsWith("Unknown descendent"));
+    }
+
+    doTopoTest(Arrays.asList("B", "D", "A", "C"),
+               Arrays.asList("D", "A", "B", "C"),
+               "B", "C",
+               "A", "B");
+
+    doTopoTest(Arrays.asList("B", "D", "A", "C"),
+               Arrays.asList("A", "D", "B", "C"),
+               "B", "C",
+               "A", "B",
+               "A", "D");
+
+    doTopoTest(Arrays.asList("B", "D", "A", "C"),
+               Arrays.asList("D", "A", "C", "B"),
+               "D", "A",
+               "C", "B");
+
+    doTopoTest(Arrays.asList("B", "D", "A", "C"),
+               Arrays.asList("D", "C", "A", "B"),
+               "D", "A",
+               "C", "B",
+               "C", "A");
+
+    doTopoTest(Arrays.asList("B", "D", "A", "C"),
+               Arrays.asList("C", "D", "A", "B"),
+               "D", "A",
+               "C", "B",
+               "C", "D");
+
+    doTopoTest(Arrays.asList("B", "D", "A", "C"),
+               Arrays.asList("D", "A", "C", "B"),
+               "D", "A",
+               "C", "B",
+               "D", "B");
+  }
+
+  private static void doTopoTest(List<String> original,
+                                 List<String> expected,
+                                 String... descs) {
+
+    List<String> values = new ArrayList<String>();
+    values.addAll(original);
+
+    TestTopoSorter tsorter = new TestTopoSorter(values, false);
+    for(int i = 0; i < descs.length; i+=2) {
+      tsorter.addDescendents(descs[i], descs[i+1]);
+    }
+
+    tsorter.sort();
+
+    assertEquals(expected, values);
+
+
+    values = new ArrayList<String>();
+    values.addAll(original);
+
+    tsorter = new TestTopoSorter(values, true);
+    for(int i = 0; i < descs.length; i+=2) {
+      tsorter.addDescendents(descs[i], descs[i+1]);
+    }
+
+    tsorter.sort();
+
+    List<String> expectedReverse = new ArrayList<String>(expected);
+    Collections.reverse(expectedReverse);
+
+    assertEquals(expectedReverse, values);
+  }
+
+  private static class TestTopoSorter extends TopoSorter<String>
+  {
+    private final Map<String,List<String>> _descMap = 
+      new HashMap<String,List<String>>();
+
+    protected TestTopoSorter(List<String> values, boolean reverse) {
+      super(values, reverse);
+    }
+
+    public void addDescendents(String from, String... tos) {
+      List<String> descs = _descMap.get(from);
+      if(descs == null) {
+        descs = new ArrayList<String>();
+        _descMap.put(from, descs);
+      }
+
+      descs.addAll(Arrays.asList(tos));
+    }
+
+    @Override
+    protected void getDescendents(String from, List<String> descendents) {
+      List<String> descs = _descMap.get(from);
+      if(descs != null) {
+        descendents.addAll(descs);
+      }
+    }
+  }
+}

src/test/java/com/healthmarketscience/jackcess/impl/expr/ExpressionatorTest.java

@@ -25,6 +25,7 @@ import com.healthmarketscience.jackcess.TestUtil;
 import com.healthmarketscience.jackcess.expr.EvalContext;
 import com.healthmarketscience.jackcess.expr.Expression;
 import com.healthmarketscience.jackcess.expr.Function;
+import com.healthmarketscience.jackcess.expr.Identifier;
 import com.healthmarketscience.jackcess.expr.TemporalConfig;
 import com.healthmarketscience.jackcess.expr.Value;
 import junit.framework.TestCase;
@@ -82,6 +83,9 @@ public class ExpressionatorTest extends TestCase
     validateExpr("IIf(\"A\",42,False)", "<EFunc>{IIf(<ELiteralValue>{\"A\"},<ELiteralValue>{42},<EConstValue>{False})}");
 
     validateExpr("\"A\" Like \"a*b\"", "<ELikeOp>{<ELiteralValue>{\"A\"} Like \"a*b\"(a.*b)}");
+
+    validateExpr("' \"A\" '", "<ELiteralValue>{\" \"\"A\"\" \"}",
+                 "\" \"\"A\"\" \"");
   }
 
   private static void doTestSimpleBinOp(String opName, String... ops) throws Exception
@@ -408,8 +412,7 @@ public class ExpressionatorTest extends TestCase
       return _thisVal;
     }
 
-    public Value getRowValue(String collectionName, String objName,
-                             String colName) {
+    public Value getIdentifierValue(Identifier identifier) {
       throw new UnsupportedOperationException();
     }