Return Input Number to Arbitrary Precision
jtigor
Function ChangePrecision(vNum, vPrecision, vPreserveSign)
Variable vNum //input number to operate on
Variable vPrecision //output precision (number of decimal places)
Variable vPreserveSign //1 == retain sign of input
Variable vReturn
Variable vExponent
Variable vMantissa
Variable vPwr
Variable vSign
//this will cause an error with the log fcn
if(vNum == 0)
return 0
endif
//precision less than 0 makes no sense
if(vPrecision < 0)
return vNum
endif
//store sign of input and ensure input is positive
vSign = sign(vNum)
vNum = abs(vNum)
//get mantissa of number
vExponent = floor(log(vNum))
vPwr = (1 *10^vExponent)
vMantissa = vNum / vPwr
//change mantissa to desired precision
vMantissa = (round(vMantissa * 10^vPrecision)) / 10^vPrecision
vReturn = (vMantissa) * vPwr
//if requested, change to original sign
if(vPreserveSign == 1)
vReturn *= vSign
endif
// Print log(vNum),vExponent, vmantissa, vPwr, vSign
return vReturn
End
Function ArbPrec(vNum, vPrecision, vPreserveSign)
Variable vNum //input number to operate on
Variable vPrecision //output precision (number of decimal places)
Variable vPreserveSign //1 == retain sign of input
Variable vSign
Variable vOutput
String sNum
//precision less than 0 makes no sense
if(vPrecision < 0)
return vNum
endif
//store sign of input and ensure input is positive
vSign = sign(vNum)
vNum *= vSign //same as abs(vNum)
//change to desired precision
APMath/N=(vPrecision) sNum = vNum
vOutput = str2num(sNum)
//if requested, change to original sign
if(vPreserveSign == 1)
vOutput *= vSign
endif
return vOutput
end
Variable vNum //input number to operate on
Variable vPrecision //output precision (number of decimal places)
Variable vPreserveSign //1 == retain sign of input
Variable vReturn
Variable vExponent
Variable vMantissa
Variable vPwr
Variable vSign
//this will cause an error with the log fcn
if(vNum == 0)
return 0
endif
//precision less than 0 makes no sense
if(vPrecision < 0)
return vNum
endif
//store sign of input and ensure input is positive
vSign = sign(vNum)
vNum = abs(vNum)
//get mantissa of number
vExponent = floor(log(vNum))
vPwr = (1 *10^vExponent)
vMantissa = vNum / vPwr
//change mantissa to desired precision
vMantissa = (round(vMantissa * 10^vPrecision)) / 10^vPrecision
vReturn = (vMantissa) * vPwr
//if requested, change to original sign
if(vPreserveSign == 1)
vReturn *= vSign
endif
// Print log(vNum),vExponent, vmantissa, vPwr, vSign
return vReturn
End
Function ArbPrec(vNum, vPrecision, vPreserveSign)
Variable vNum //input number to operate on
Variable vPrecision //output precision (number of decimal places)
Variable vPreserveSign //1 == retain sign of input
Variable vSign
Variable vOutput
String sNum
//precision less than 0 makes no sense
if(vPrecision < 0)
return vNum
endif
//store sign of input and ensure input is positive
vSign = sign(vNum)
vNum *= vSign //same as abs(vNum)
//change to desired precision
APMath/N=(vPrecision) sNum = vNum
vOutput = str2num(sNum)
//if requested, change to original sign
if(vPreserveSign == 1)
vOutput *= vSign
endif
return vOutput
end
Forum
Support
Gallery
Igor Pro 9
Learn More
Igor XOP Toolkit
Learn More
Igor NIDAQ Tools MX
Learn More