normalize decimals when used; convert number literals to decimal instead of double; handle int overflow for number literals

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/branches/exprs@1155 f203690c-595d-4dc9-a70b-905162fa7fd2

5 files changed, 90 insertions, 44 deletions

diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/expr/BuiltinOperators.java b/src/main/java/com/healthmarketscience/jackcess/impl/expr/BuiltinOperators.java
index 596f36e..8400a7c 100644
--- a/src/main/java/com/healthmarketscience/jackcess/impl/expr/BuiltinOperators.java
+++ b/src/main/java/com/healthmarketscience/jackcess/impl/expr/BuiltinOperators.java
@@ -58,6 +58,7 @@ public class BuiltinOperators
   public static final Value ZERO_VAL = FALSE_VAL;
 
   public static final RoundingMode ROUND_MODE = RoundingMode.HALF_EVEN;
+  private static final int MAX_NUMERIC_SCALE = 28;
 
   private enum CoercionType {
     SIMPLE(true, true), GENERAL(false, true), COMPARE(false, false);
@@ -223,7 +224,7 @@ public class BuiltinOperators
       }
       return toValue(param1.getAsDouble() / d2);
     case BIG_DEC:
-      return toValue(param1.getAsBigDecimal().divide(param2.getAsBigDecimal()));
+      return toValue(divide(param1.getAsBigDecimal(), param2.getAsBigDecimal()));
     default:
       throw new EvalException("Unexpected type " + mathType);
     }
@@ -582,12 +583,12 @@ public class BuiltinOperators
   }
 
   public static Value toValue(BigDecimal s) {
-    return new BigDecimalValue(s);
+    return new BigDecimalValue(normalize(s));
   }
 
   public static Value toValue(Value.Type type, double dd, DateFormat fmt) {
-    return toValue(type, new Date(ColumnImpl.fromDateDouble(dd, fmt.getCalendar())),
-                   fmt);
+    return toValue(type, new Date(ColumnImpl.fromDateDouble(
+                                      dd, fmt.getCalendar())), fmt);
   }
 
   public static Value toValue(EvalContext ctx, Value.Type type, Date d) {
@@ -730,7 +731,7 @@ public class BuiltinOperators
       if(prefType.isNumeric()) {
         // re-evaluate the numeric type choice based on the type of the parsed
         // number
-        Value.Type numType = ((num.stripTrailingZeros().scale() > 0) ?
+        Value.Type numType = ((num.scale() > 0) ?
                               Value.Type.BIG_DEC : Value.Type.LONG);
         prefType = getPreferredNumericType(numType, prefType);
       }
@@ -746,10 +747,31 @@ public class BuiltinOperators
     return ((t1.compareTo(t2) > 0) ? t1 : t2);
   }
 
+  static BigDecimal divide(BigDecimal num, BigDecimal denom) {
+    return num.divide(denom, MAX_NUMERIC_SCALE, ROUND_MODE);
+  }
+
   static boolean isIntegral(double d) {
     double id = Math.rint(d);
     return ((d == id) && (d >= MIN_INT) && (d <= MAX_INT) &&
             !Double.isInfinite(d) && !Double.isNaN(d));
   }
 
+  /**
+   * Converts the given BigDecimal to the minimal scale >= 0;
+   */
+  static BigDecimal normalize(BigDecimal bd) {
+    if(bd.scale() == 0) {
+      return bd;
+    }
+    // handle a bug in the jdk which doesn't strip zero values
+    if(bd.compareTo(BigDecimal.ZERO) == 0) {
+      return BigDecimal.ZERO;
+    }
+    bd = bd.stripTrailingZeros();
+    if(bd.scale() < 0) {
+      bd = bd.setScale(0);
+    }
+    return bd;
+  }
 }
diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/expr/ExpressionTokenizer.java b/src/main/java/com/healthmarketscience/jackcess/impl/expr/ExpressionTokenizer.java
index 89e2049..2b64da5 100644
--- a/src/main/java/com/healthmarketscience/jackcess/impl/expr/ExpressionTokenizer.java
+++ b/src/main/java/com/healthmarketscience/jackcess/impl/expr/ExpressionTokenizer.java
@@ -16,6 +16,7 @@ limitations under the License.
 
 package com.healthmarketscience.jackcess.impl.expr;
 
+import java.math.BigDecimal;
 import java.text.DateFormat;
 import java.text.FieldPosition;
 import java.text.ParsePosition;
@@ -40,7 +41,7 @@ import com.healthmarketscience.jackcess.expr.ParseException;
  *
  * @author James Ahlborn
  */
-class ExpressionTokenizer 
+class ExpressionTokenizer
 {
   private static final int EOF = -1;
   private static final char QUOTED_STR_CHAR = '"';
@@ -100,10 +101,10 @@ class ExpressionTokenizer
 
     while(buf.hasNext()) {
       char c = buf.next();
-      
+
       byte charFlag = getCharFlag(c);
       if(charFlag != 0) {
-        
+
         // what could it be?
         switch(charFlag) {
         case IS_OP_FLAG:
@@ -115,14 +116,14 @@ class ExpressionTokenizer
         case IS_COMP_FLAG:
 
           // special case for default values
-          if((exprType == Type.DEFAULT_VALUE) && (c == EQUALS_CHAR) && 
+          if((exprType == Type.DEFAULT_VALUE) && (c == EQUALS_CHAR) &&
              (buf.prevPos() == 0)) {
             // a leading equals sign indicates how a default value should be
             // evaluated
             tokens.add(new Token(TokenType.OP, String.valueOf(c)));
             continue;
           }
-          
+
           tokens.add(new Token(TokenType.OP, parseCompOp(c, buf)));
           break;
 
@@ -144,7 +145,7 @@ class ExpressionTokenizer
           switch(c) {
           case QUOTED_STR_CHAR:
           case SINGLE_QUOTED_STR_CHAR:
-            tokens.add(new Token(TokenType.LITERAL, null, 
+            tokens.add(new Token(TokenType.LITERAL, null,
                                  parseQuotedString(buf, c), Value.Type.STRING));
             break;
           case DATE_LIT_QUOTE_CHAR:
@@ -211,12 +212,12 @@ class ExpressionTokenizer
 
   private static void consumeWhitespace(ExprBuf buf) {
     int c = EOF;
-    while(((c = buf.peekNext()) != EOF) && 
+    while(((c = buf.peekNext()) != EOF) &&
           hasFlag(getCharFlag((char)c), IS_SPACE_FLAG)) {
         buf.next();
     }
   }
-  
+
   private static String parseBareString(char firstChar, ExprBuf buf,
                                         Type exprType) {
     StringBuilder sb = buf.getScratchBuffer().append(firstChar);
@@ -235,7 +236,7 @@ class ExpressionTokenizer
       }
       sb.append(c);
     }
-    
+
     return sb.toString();
   }
 
@@ -251,9 +252,9 @@ class ExpressionTokenizer
     return parseStringUntil(buf, DATE_LIT_QUOTE_CHAR, null, false);
   }
 
-  private static String parseStringUntil(ExprBuf buf, char endChar, 
+  private static String parseStringUntil(ExprBuf buf, char endChar,
                                          Character startChar,
-                                         boolean allowDoubledEscape) 
+                                         boolean allowDoubledEscape)
   {
     StringBuilder sb = buf.getScratchBuffer();
 
@@ -285,21 +286,21 @@ class ExpressionTokenizer
     return sb.toString();
   }
 
-  private static Token parseDateLiteral(ExprBuf buf) 
+  private static Token parseDateLiteral(ExprBuf buf)
   {
     TemporalConfig cfg = buf.getTemporalConfig();
     String dateStr = parseDateLiteralString(buf);
-    
+
     boolean hasDate = (dateStr.indexOf(cfg.getDateSeparator()) >= 0);
     boolean hasTime = (dateStr.indexOf(cfg.getTimeSeparator()) >= 0);
     boolean hasAmPm = false;
-    
+
     if(hasTime) {
       int strLen = dateStr.length();
       hasAmPm = ((strLen >= AMPM_SUFFIX_LEN) &&
-                 (dateStr.regionMatches(true, strLen - AMPM_SUFFIX_LEN, 
+                 (dateStr.regionMatches(true, strLen - AMPM_SUFFIX_LEN,
                                         AM_SUFFIX, 0, AMPM_SUFFIX_LEN) ||
-                  dateStr.regionMatches(true, strLen - AMPM_SUFFIX_LEN, 
+                  dateStr.regionMatches(true, strLen - AMPM_SUFFIX_LEN,
                                         PM_SUFFIX, 0, AMPM_SUFFIX_LEN)));
     }
 
@@ -323,7 +324,7 @@ class ExpressionTokenizer
       return new Token(TokenType.LITERAL, sdf.parse(dateStr), dateStr, valType,
                        sdf);
     } catch(java.text.ParseException pe) {
-      throw new ParseException(       
+      throw new ParseException(
           "Invalid date time literal " + dateStr + " " + buf, pe);
     }
   }
@@ -372,18 +373,32 @@ class ExpressionTokenizer
 
       String numStr = sb.toString();
       try {
-        // what number type to use here?
-        Object num = (isFp ? 
-                      (Number)Double.valueOf(numStr) : 
-                      (Number)Integer.valueOf(numStr));
+        Number num = null;
+        Value.Type numType = null;
+
+        if(!isFp) {
+          try {
+            // try to parse as int.  if that fails, fall back to BigDecimal
+            // (this will handle the case of int overflow)
+            num = Integer.valueOf(numStr);
+            numType = Value.Type.LONG;
+          } catch(NumberFormatException ne) {
+            // fallback to decimal
+          }
+        }
+
+        if(num == null) {
+          num = new BigDecimal(numStr);
+          numType = Value.Type.BIG_DEC;
+        }
+
         foundNum = true;
-        return new Token(TokenType.LITERAL, num, numStr, 
-                         (isFp ? Value.Type.DOUBLE : Value.Type.LONG));
+        return new Token(TokenType.LITERAL, num, numStr, numType);
       } catch(NumberFormatException ne) {
         throw new ParseException(
             "Invalid number literal " + numStr + " " + buf, ne);
       }
-      
+
     } finally {
       if(!foundNum) {
         buf.reset(startPos);
@@ -421,7 +436,7 @@ class ExpressionTokenizer
     private DateFormat _dateTimeFmt24;
     private String _baseDate;
     private final StringBuilder _scratch = new StringBuilder();
-    
+
     private ExprBuf(String str, ParseContext ctx) {
       _str = str;
       _ctx = ctx;
@@ -609,7 +624,7 @@ class ExpressionTokenizer
     private final DateFormat _fmtDelegate;
     private final String _baseDate;
 
-    private TimeFormat(DateFormat parseDelegate, DateFormat fmtDelegate, 
+    private TimeFormat(DateFormat parseDelegate, DateFormat fmtDelegate,
                        String baseDate)
     {
       _parseDelegate = parseDelegate;
diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/expr/Expressionator.java b/src/main/java/com/healthmarketscience/jackcess/impl/expr/Expressionator.java
index 2c8879f..1935149 100644
--- a/src/main/java/com/healthmarketscience/jackcess/impl/expr/Expressionator.java
+++ b/src/main/java/com/healthmarketscience/jackcess/impl/expr/Expressionator.java
@@ -1331,7 +1331,7 @@ public class Expressionator
   {
     switch(valType) {
     case STRING:
-      return new StringValue((String)value);
+      return BuiltinOperators.toValue((String)value);
     case DATE:
       return new DateValue((Date)value, sdf);
     case TIME:
@@ -1339,11 +1339,11 @@ public class Expressionator
     case DATE_TIME:
       return new DateTimeValue((Date)value, sdf);
     case LONG:
-      return new LongValue((Integer)value);
+      return BuiltinOperators.toValue((Integer)value);
     case DOUBLE:
-      return new DoubleValue((Double)value);
+      return BuiltinOperators.toValue((Double)value);
     case BIG_DEC:
-      return new BigDecimalValue((BigDecimal)value);
+      return BuiltinOperators.toValue((BigDecimal)value);
     default:
       throw new ParseException("unexpected literal type " + valType);
     }
diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/expr/StringValue.java b/src/main/java/com/healthmarketscience/jackcess/impl/expr/StringValue.java
index 9f2a295..014e371 100644
--- a/src/main/java/com/healthmarketscience/jackcess/impl/expr/StringValue.java
+++ b/src/main/java/com/healthmarketscience/jackcess/impl/expr/StringValue.java
@@ -75,7 +75,7 @@ public class StringValue extends BaseValue
     if(_num == null) {
       // see if it is parseable as a number
       try {
-        _num = new BigDecimal(_val);
+        _num = BuiltinOperators.normalize(new BigDecimal(_val));
         return (BigDecimal)_num;
       } catch(NumberFormatException nfe) {
         _num = NOT_A_NUMBER;
diff --git a/src/test/java/com/healthmarketscience/jackcess/impl/expr/ExpressionatorTest.java b/src/test/java/com/healthmarketscience/jackcess/impl/expr/ExpressionatorTest.java
index f659a86..f20179a 100644
--- a/src/test/java/com/healthmarketscience/jackcess/impl/expr/ExpressionatorTest.java
+++ b/src/test/java/com/healthmarketscience/jackcess/impl/expr/ExpressionatorTest.java
@@ -148,11 +148,11 @@ public class ExpressionatorTest extends TestCase
     }
 
     for(double i : DBLS) {
-      assertEquals(-i, eval("=-(" + i + ")"));
+      assertEquals(toBD(-i), eval("=-(" + i + ")"));
     }
 
     for(double i : DBLS) {
-      assertEquals(i, eval("=+(" + i + ")"));
+      assertEquals(toBD(i), eval("=+(" + i + ")"));
     }
 
     for(int i = -10; i <= 10; ++i) {
@@ -163,7 +163,7 @@ public class ExpressionatorTest extends TestCase
 
     for(double i : DBLS) {
       for(double j : DBLS) {
-        assertEquals((i + j), eval("=" + i + " + " + j));
+        assertEquals(toBD(toBD(i).add(toBD(j))), eval("=" + i + " + " + j));
       }
     }
 
@@ -175,7 +175,7 @@ public class ExpressionatorTest extends TestCase
 
     for(double i : DBLS) {
       for(double j : DBLS) {
-        assertEquals((i - j), eval("=" + i + " - " + j));
+        assertEquals(toBD(toBD(i).subtract(toBD(j))), eval("=" + i + " - " + j));
       }
     }
 
@@ -187,7 +187,7 @@ public class ExpressionatorTest extends TestCase
 
     for(double i : DBLS) {
       for(double j : DBLS) {
-        assertEquals((i * j), eval("=" + i + " * " + j));
+        assertEquals(toBD(toBD(i).multiply(toBD(j))), eval("=" + i + " * " + j));
       }
     }
 
@@ -253,7 +253,8 @@ public class ExpressionatorTest extends TestCase
         if(j == 0.0d) {
           evalFail("=" + i + " / " + j, ArithmeticException.class);
         } else {
-          assertEquals((i / j), eval("=" + i + " / " + j));
+          assertEquals(toBD(BuiltinOperators.divide(toBD(i), toBD(j))),
+                       eval("=" + i + " / " + j));
         }
       }
     }
@@ -282,8 +283,8 @@ public class ExpressionatorTest extends TestCase
     assertEquals(-100, eval("=-(10)^2"));
     assertEquals(-100, eval("=-\"10\"^2"));
 
-    assertEquals(99d, eval("=-10E-1+10e+1"));
-    assertEquals(-101d, eval("=-10E-1-10e+1"));
+    assertEquals(toBD(99d), eval("=-10E-1+10e+1"));
+    assertEquals(toBD(-101d), eval("=-10E-1-10e+1"));
   }
 
   public void testTypeCoercion() throws Exception
@@ -368,6 +369,14 @@ public class ExpressionatorTest extends TestCase
       .intValueExact();
   }
 
+  static BigDecimal toBD(double d) {
+    return toBD(new BigDecimal("" + d));
+  }
+
+  static BigDecimal toBD(BigDecimal bd) {
+    return BuiltinOperators.normalize(bd);
+  }
+
   private static final class TestParseContext implements Expressionator.ParseContext
   {
     public TemporalConfig getTemporalConfig() {