Bug 60405: Add initial support for cetab functions so some macros can be

parsed
Add some function-definitions for Excel 4 Macros and missing functions
found in regression tests

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1852277 13f79535-47bb-0310-9956-ffa450edef68

7 files changed, 116 insertions, 59 deletions

diff --git a/src/java/org/apache/poi/ss/formula/function/FunctionDataBuilder.java b/src/java/org/apache/poi/ss/formula/function/FunctionDataBuilder.java
index c535bf4ca2..0be53f555e 100644
--- a/src/java/org/apache/poi/ss/formula/function/FunctionDataBuilder.java
+++ b/src/java/org/apache/poi/ss/formula/function/FunctionDataBuilder.java
@@ -81,8 +81,7 @@ final class FunctionDataBuilder {
 		FunctionMetadata[] jumbledArray =  new FunctionMetadata[_functionDataByName.size()];
 		_functionDataByName.values().toArray(jumbledArray);
 		FunctionMetadata[] fdIndexArray = new FunctionMetadata[_maxFunctionIndex+1];
-		for (int i = 0; i < jumbledArray.length; i++) {
-			FunctionMetadata fd = jumbledArray[i];
+		for (FunctionMetadata fd : jumbledArray) {
 			fdIndexArray[fd.getIndex()] = fd;
 		}
 
diff --git a/src/java/org/apache/poi/ss/formula/function/FunctionMetadata.java b/src/java/org/apache/poi/ss/formula/function/FunctionMetadata.java
index 6ab06f0ad8..fa8ca34aca 100644
--- a/src/java/org/apache/poi/ss/formula/function/FunctionMetadata.java
+++ b/src/java/org/apache/poi/ss/formula/function/FunctionMetadata.java
@@ -31,6 +31,7 @@ public final class FunctionMetadata {
 	 * to make that file more version neutral.
 	 * @see org.apache.poi.ss.formula.FormulaParser#validateNumArgs(int, FunctionMetadata)
 	 */
+	@SuppressWarnings("JavadocReference")
 	private static final short FUNCTION_MAX_PARAMS = 30;
 
 	private final int _index;
@@ -49,27 +50,35 @@ public final class FunctionMetadata {
 		_returnClassCode = returnClassCode;
 		_parameterClassCodes = (parameterClassCodes == null) ? null : parameterClassCodes.clone();
 	}
+
 	public int getIndex() {
 		return _index;
 	}
+
 	public String getName() {
 		return _name;
 	}
+
 	public int getMinParams() {
 		return _minParams;
 	}
+
 	public int getMaxParams() {
 		return _maxParams;
 	}
+
 	public boolean hasFixedArgsLength() {
 		return _minParams == _maxParams;
 	}
+
 	public byte getReturnClassCode() {
 		return _returnClassCode;
 	}
+
 	public byte[] getParameterClassCodes() {
 		return _parameterClassCodes.clone();
 	}
+
 	/**
 	 * Some varags functions (like VLOOKUP) have a specific limit to the number of arguments that 
 	 * can be passed.  Other functions (like SUM) don't have such a limit.  For those functions,
@@ -80,11 +89,8 @@ public final class FunctionMetadata {
 	public boolean hasUnlimitedVarags() {
 		return FUNCTION_MAX_PARAMS == _maxParams;
 	}
+
 	public String toString() {
-		StringBuffer sb = new StringBuffer(64);
-		sb.append(getClass().getName()).append(" [");
-		sb.append(_index).append(" ").append(_name);
-		sb.append("]");
-		return sb.toString();
+		return getClass().getName() + " [" + _index + " " + _name + "]";
 	}
 }
diff --git a/src/java/org/apache/poi/ss/formula/function/FunctionMetadataReader.java b/src/java/org/apache/poi/ss/formula/function/FunctionMetadataReader.java
index dc6a8c0255..dc83e9c601 100644
--- a/src/java/org/apache/poi/ss/formula/function/FunctionMetadataReader.java
+++ b/src/java/org/apache/poi/ss/formula/function/FunctionMetadataReader.java
@@ -41,6 +41,7 @@ final class FunctionMetadataReader {
 	private static final int MAX_RECORD_LENGTH = 100_000;
 
 	private static final String METADATA_FILE_NAME = "functionMetadata.txt";
+	private static final String METADATA_FILE_NAME_CETAB = "functionMetadataCetab.txt";
 
 	/** plain ASCII text metadata file uses three dots for ellipsis */
 	private static final String ELLIPSIS = "...";
@@ -52,51 +53,56 @@ final class FunctionMetadataReader {
 	private static final String[] DIGIT_ENDING_FUNCTION_NAMES = {
 		// Digits at the end of a function might be due to a left-over footnote marker.
 		// except in these cases
-		"LOG10", "ATAN2", "DAYS360", "SUMXMY2", "SUMX2MY2", "SUMX2PY2",
+		"LOG10", "ATAN2", "DAYS360", "SUMXMY2", "SUMX2MY2", "SUMX2PY2", "A1.R1C1",
 	};
 	private static final Set<String> DIGIT_ENDING_FUNCTION_NAMES_SET = new HashSet<>(Arrays.asList(DIGIT_ENDING_FUNCTION_NAMES));
 
 	public static FunctionMetadataRegistry createRegistry() {
-	    try {
-    		InputStream is = FunctionMetadataReader.class.getResourceAsStream(METADATA_FILE_NAME);
-    		if (is == null) {
-    			throw new RuntimeException("resource '" + METADATA_FILE_NAME + "' not found");
-    		}
-    
-    		try {
-        		try(BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
-        		    FunctionDataBuilder fdb = new FunctionDataBuilder(400);
-        
-        			while (true) {
-        				String line = br.readLine();
-        				if (line == null) {
-        					break;
-        				}
-        				if (line.length() < 1 || line.charAt(0) == '#') {
-        					continue;
-        				}
-        				String trimLine = line.trim();
-        				if (trimLine.length() < 1) {
-        					continue;
-        				}
-        				processLine(fdb, line);
-        			}
-
-        			return fdb.build();
-        		}
-    		} finally {
-    		    is.close();
-    		}
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-	    } 
+		FunctionDataBuilder fdb = new FunctionDataBuilder(800);
+		readResourceFile(fdb, METADATA_FILE_NAME);
+		return fdb.build();
+	}
+
+	public static FunctionMetadataRegistry createRegistryCetab() {
+		FunctionDataBuilder fdb = new FunctionDataBuilder(800);
+		readResourceFile(fdb, METADATA_FILE_NAME_CETAB);
+		return fdb.build();
+	}
+
+	private static void readResourceFile(FunctionDataBuilder fdb, String resourceFile) {
+		try (InputStream is = FunctionMetadataReader.class.getResourceAsStream(resourceFile)) {
+			if (is == null) {
+				throw new RuntimeException("resource '" + resourceFile + "' not found");
+			}
+
+			try(BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
+
+				while (true) {
+					String line = br.readLine();
+					if (line == null) {
+						break;
+					}
+					if (line.length() < 1 || line.charAt(0) == '#') {
+						continue;
+					}
+					String trimLine = line.trim();
+					if (trimLine.length() < 1) {
+						continue;
+					}
+					processLine(fdb, line);
+				}
+			}
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
 	}
 
 	private static void processLine(FunctionDataBuilder fdb, String line) {
 
 		String[] parts = TAB_DELIM_PATTERN.split(line, -2);
 		if(parts.length != 8) {
-			throw new RuntimeException("Bad line format '" + line + "' - expected 8 data fields");
+			throw new RuntimeException("Bad line format '" + line + "' - expected 8 data fields delimited by tab, " +
+					"but had " + parts.length + ": " + Arrays.toString(parts));
 		}
 		int functionIndex = parseInt(parts[0]);
 		String functionName = parts[1];
diff --git a/src/java/org/apache/poi/ss/formula/function/FunctionMetadataRegistry.java b/src/java/org/apache/poi/ss/formula/function/FunctionMetadataRegistry.java
index 34317fe6d8..bb01b41163 100644
--- a/src/java/org/apache/poi/ss/formula/function/FunctionMetadataRegistry.java
+++ b/src/java/org/apache/poi/ss/formula/function/FunctionMetadataRegistry.java
@@ -37,6 +37,7 @@ public final class FunctionMetadataRegistry {
 	public static final short FUNCTION_INDEX_EXTERNAL = 255;
 
 	private static FunctionMetadataRegistry _instance;
+	private static FunctionMetadataRegistry _instanceCetab;
 
 	private final FunctionMetadata[] _functionDataByIndex;
 	private final Map<String, FunctionMetadata> _functionDataByName;
@@ -48,6 +49,13 @@ public final class FunctionMetadataRegistry {
 		return _instance;
 	}
 
+	private static FunctionMetadataRegistry getInstanceCetab() {
+		if (_instanceCetab == null) {
+			_instanceCetab = FunctionMetadataReader.createRegistryCetab();
+		}
+		return _instanceCetab;
+	}
+
 	/* package */ FunctionMetadataRegistry(FunctionMetadata[] functionDataByIndex, Map<String, FunctionMetadata> functionDataByName) {
 		_functionDataByIndex = (functionDataByIndex == null) ? null : functionDataByIndex.clone();
 		_functionDataByName = functionDataByName;
@@ -62,6 +70,10 @@ public final class FunctionMetadataRegistry {
 		return getInstance().getFunctionByIndexInternal(index);
 	}
 
+	public static FunctionMetadata getCetabFunctionByIndex(int index) {
+		return getInstanceCetab().getFunctionByIndexInternal(index);
+	}
+
 	private FunctionMetadata getFunctionByIndexInternal(int index) {
 		return _functionDataByIndex[index];
 	}
@@ -74,7 +86,11 @@ public final class FunctionMetadataRegistry {
 	public static short lookupIndexByName(String name) {
 		FunctionMetadata fd = getInstance().getFunctionByNameInternal(name);
 		if (fd == null) {
-			return -1;
+			// also try the cetab functions
+			fd = getInstanceCetab().getFunctionByNameInternal(name);
+			if (fd == null) {
+				return -1;
+			}
 		}
 		return (short) fd.getIndex();
 	}
@@ -83,8 +99,12 @@ public final class FunctionMetadataRegistry {
 		return _functionDataByName.get(name);
 	}
 
-
 	public static FunctionMetadata getFunctionByName(String name) {
-		return getInstance().getFunctionByNameInternal(name);
+		FunctionMetadata fm = getInstance().getFunctionByNameInternal(name);
+		if(fm == null) {
+			return getInstanceCetab().getFunctionByNameInternal(name);
+		}
+
+		return fm;
 	}
 }
diff --git a/src/java/org/apache/poi/ss/formula/ptg/AbstractFunctionPtg.java b/src/java/org/apache/poi/ss/formula/ptg/AbstractFunctionPtg.java
index 28b5e7d9e5..bfb7aeba6a 100644
--- a/src/java/org/apache/poi/ss/formula/ptg/AbstractFunctionPtg.java
+++ b/src/java/org/apache/poi/ss/formula/ptg/AbstractFunctionPtg.java
@@ -123,13 +123,22 @@ public abstract class AbstractFunctionPtg extends OperationPtg {
         return ix >= 0;
     }
 
-    protected final String lookupName(short index) {
+    protected String lookupName(short index) {
+        return lookupName(index, false);
+    }
+
+    protected final String lookupName(short index, boolean isCetab) {
         if(index == FunctionMetadataRegistry.FUNCTION_INDEX_EXTERNAL) {
             return "#external#";
         }
-        FunctionMetadata fm = FunctionMetadataRegistry.getFunctionByIndex(index);
+        final FunctionMetadata fm;
+        if(isCetab) {
+            fm = FunctionMetadataRegistry.getCetabFunctionByIndex(index);
+        } else {
+            fm = FunctionMetadataRegistry.getFunctionByIndex(index);
+        }
         if(fm == null) {
-            throw new RuntimeException("bad function index (" + index + ")");
+            throw new RuntimeException("bad function index (" + index + ", " + isCetab + ")");
         }
         return fm.getName();
     }
diff --git a/src/java/org/apache/poi/ss/formula/ptg/ExpPtg.java b/src/java/org/apache/poi/ss/formula/ptg/ExpPtg.java
index d128be9951..83ebc2044d 100644
--- a/src/java/org/apache/poi/ss/formula/ptg/ExpPtg.java
+++ b/src/java/org/apache/poi/ss/formula/ptg/ExpPtg.java
@@ -69,9 +69,6 @@ public final class ExpPtg extends ControlPtg {
 
     @Override
     public String toString() {
-        StringBuffer buffer = new StringBuffer("[Array Formula or Shared Formula]\n");
-        buffer.append("row = ").append(getRow()).append("\n");
-        buffer.append("col = ").append(getColumn()).append("\n");
-        return buffer.toString();
+        return "[Array Formula or Shared Formula]\n" + "row = " + getRow() + "\n" + "col = " + getColumn() + "\n";
     }
 }
diff --git a/src/java/org/apache/poi/ss/formula/ptg/FuncVarPtg.java b/src/java/org/apache/poi/ss/formula/ptg/FuncVarPtg.java
index 3392d04a76..a1cacd8ba5 100644
--- a/src/java/org/apache/poi/ss/formula/ptg/FuncVarPtg.java
+++ b/src/java/org/apache/poi/ss/formula/ptg/FuncVarPtg.java
@@ -18,6 +18,8 @@
 package org.apache.poi.ss.formula.ptg;
 import org.apache.poi.ss.formula.function.FunctionMetadata;
 import org.apache.poi.ss.formula.function.FunctionMetadataRegistry;
+import org.apache.poi.util.BitField;
+import org.apache.poi.util.BitFieldFactory;
 import org.apache.poi.util.LittleEndianInput;
 import org.apache.poi.util.LittleEndianOutput;
 
@@ -25,24 +27,29 @@ import org.apache.poi.util.LittleEndianOutput;
  * @author Jason Height (jheight at chariot dot net dot au)
  */
 public final class FuncVarPtg extends AbstractFunctionPtg{
-
     public final static byte sid  = 0x22;
     private final static int  SIZE = 4;
 
+    // See spec at 2.5.198.63 PtgFuncVar
+    private static final BitField ceFunc = BitFieldFactory.getInstance(0xF000);
+
     /**
      * Single instance of this token for 'sum() taking a single argument'
      */
     public static final OperationPtg SUM = FuncVarPtg.create("SUM", 1);
 
-    private FuncVarPtg(int functionIndex, int returnClass, byte[] paramClasses, int numArgs) {
+    private final boolean _isCetab;
+
+    private FuncVarPtg(int functionIndex, int returnClass, byte[] paramClasses, int numArgs, boolean isCetab) {
         super(functionIndex, returnClass, paramClasses, numArgs);
+        _isCetab = isCetab;
     }
 
     /**Creates new function pointer from a byte array
      * usually called while reading an excel file.
      */
     public static FuncVarPtg create(LittleEndianInput in)  {
-        return create(in.readByte(), in.readShort());
+        return create(in.readByte(), in.readUShort());
     }
 
     /**
@@ -53,12 +60,25 @@ public final class FuncVarPtg extends AbstractFunctionPtg{
     }
 
     private static FuncVarPtg create(int numArgs, int functionIndex) {
-        FunctionMetadata fm = FunctionMetadataRegistry.getFunctionByIndex(functionIndex);
-        if(fm == null) {
+        final FunctionMetadata fm;
+        boolean isCetab = ceFunc.isSet(functionIndex);
+        if(isCetab) {
+            functionIndex = ceFunc.clear(functionIndex);
+            fm = FunctionMetadataRegistry.getCetabFunctionByIndex(functionIndex);
+        } else {
+            fm = FunctionMetadataRegistry.getFunctionByIndex(functionIndex);
+        }
+
+        if (fm == null) {
             // Happens only as a result of a call to FormulaParser.parse(), with a non-built-in function name
-            return new FuncVarPtg(functionIndex, Ptg.CLASS_VALUE, new byte[] {Ptg.CLASS_VALUE}, numArgs);
+            return new FuncVarPtg(functionIndex, Ptg.CLASS_VALUE, new byte[]{Ptg.CLASS_VALUE}, numArgs, isCetab);
         }
-        return new FuncVarPtg(functionIndex, fm.getReturnClassCode(), fm.getParameterClassCodes(), numArgs);
+        return new FuncVarPtg(functionIndex, fm.getReturnClassCode(), fm.getParameterClassCodes(), numArgs, isCetab);
+    }
+
+    @Override
+    protected String lookupName(short index) {
+        return lookupName(index, _isCetab);
     }
 
     public void write(LittleEndianOutput out) {