Skip to content

fix(db): export numbers at DDL scale instead of a fixed two decimals #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
karlkauc merged 1 commit into from
Aug 30, 2026
Merged

fix(db): export numbers at DDL scale instead of a fixed two decimals #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Database_Integration/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ portfolios, multiple positions** and preserve their order, so the export is
deterministic and faithful. SQLite makes the examples zero-setup; the same SQL
runs on PostgreSQL (Oracle/SQL Server need only type tweaks — noted in the DDL).

### Numeric precision on export

All four exporters render numbers at the **DDL scale** (`schema.sql`:
amounts `DECIMAL(20,2)`, `TotalPercentage` `DECIMAL(9,4)`, quantities /
`NavPrice` / `SharesOutstanding` `DECIMAL(28,6)`), then trim trailing zeros
down to two decimals (zero for `SharesOutstanding`). So `8.33` stays `8.33`,
`8.3333` stays `8.3333` and `50000.123456` units survive the round-trip;
values with *more* decimals than the DDL allows are rounded on export
(`xml_equiv.py` compares numerically and would report it).

## Round-trip is proven by equivalence

The user requirement: export the data the import wrote, then check the import
Expand Down
28 changes: 22 additions & 6 deletions Database_Integration/csharp/export/ExportFundsXml.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,24 @@ internal static class ExportFundsXml
["ShareClass"] = "Shares", ["Option"] = "Contracts",
["Future"] = "Contracts" };

static string Inv(double v) => v.ToString("0.00", CultureInfo.InvariantCulture);
/// <summary>Amounts: DDL scale 2 (see Num).</summary>
static string Inv(double v) => Num(v, 2, 2);

/// <summary>
/// Number formatting follows the DDL scale (schema.sql): amounts
/// DECIMAL(20,2), TotalPercentage DECIMAL(9,4), quantities / NavPrice /
/// SharesOutstanding DECIMAL(28,6). Render at that scale, then drop
/// trailing zeros down to a floor of <paramref name="minDec"/> decimals
/// (custom format "0.00####"): 8.33 -> "8.33", 8.3333 -> "8.3333",
/// 550000 shares -> "550000". A fixed "0.00" would silently truncate what
/// the model stores (xml_equiv.py compares numerically and would flag it).
/// </summary>
static string Num(double v, int scale, int minDec)
{
decimal d = Math.Round((decimal)v, scale, MidpointRounding.AwayFromZero);
string fmt = "0" + (scale > 0 ? "." + new string('0', minDec) + new string('#', scale - minDec) : "");
return d.ToString(fmt, CultureInfo.InvariantCulture);
}

static XmlElement El(XmlDocument doc, XmlNode parent, string tag,
string? text = null)
Expand Down Expand Up @@ -198,12 +215,12 @@ static int Main(string[] args)
var tv = El(doc, El(doc, pos, "TotalValue"), "Amount",
Inv((double)p["value"]!));
tv.SetAttribute("ccy", ccy);
El(doc, pos, "TotalPercentage", Inv((double)p["pct"]!));
El(doc, pos, "TotalPercentage", Num((double)p["pct"]!, 4, 2));
var kind = p["kind"] is string k
&& PositionKinds.Contains(k) ? k : "Generic";
var ke = El(doc, pos, kind);
if (QtyElem.ContainsKey(kind) && p["qty"] != null)
El(doc, ke, QtyElem[kind], Inv((double)p["qty"]!));
El(doc, ke, QtyElem[kind], Num((double)p["qty"]!, 6, 2));
}
}

Expand Down Expand Up @@ -245,7 +262,7 @@ static int Main(string[] args)
(string?)sc["currency"]);
El(doc, pr2, "PriceNature", "OFFICIAL");
El(doc, pr2, "NavPrice",
Inv((double)sc["nav_price"]!));
Num((double)sc["nav_price"]!, 6, 2));
}
if (sc["nav_fund_ccy"] != null)
{
Expand All @@ -258,8 +275,7 @@ static int Main(string[] args)
a2.SetAttribute("ccy", ccy);
if (sc["shares"] != null)
El(doc, t2, "SharesOutstanding",
((double)sc["shares"]!).ToString("0",
CultureInfo.InvariantCulture));
Num((double)sc["shares"]!, 6, 0));
}
}
}
Expand Down
30 changes: 23 additions & 7 deletions Database_Integration/java/ExportFundsXml.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,24 @@ public class ExportFundsXml {
"Bond", "Nominal", "ShareClass", "Shares",
"Option", "Contracts", "Future", "Contracts");

static String f2(double v) { return String.format(Locale.ROOT, "%.2f", v); }
/** Amounts: DDL scale 2 (see num). */
static String f2(double v) { return num(v, 2, 2); }

/**
* Number formatting follows the DDL scale (schema.sql): amounts
* DECIMAL(20,2), TotalPercentage DECIMAL(9,4), quantities / NavPrice /
* SharesOutstanding DECIMAL(28,6). Render at that scale, then drop trailing
* zeros down to a floor of {@code minDec} decimals: 8.33 -> "8.33",
* 8.3333 -> "8.3333", 550000 shares -> "550000". A fixed "%.2f" would
* silently truncate what the model can store (xml_equiv.py compares
* numerically and would flag the loss).
*/
static String num(double v, int scale, int minDec) {
java.math.BigDecimal d = java.math.BigDecimal.valueOf(v)
.setScale(scale, java.math.RoundingMode.HALF_UP).stripTrailingZeros();
if (d.scale() < minDec) d = d.setScale(minDec);
return d.toPlainString();
}

/** Append <tag>text</tag> to parent (the one XML-build primitive). */
static Element el(Document doc, org.w3c.dom.Node parent, String tag,
Expand Down Expand Up @@ -161,15 +178,15 @@ public static void main(String[] args) throws Exception {
"Amount", f2(qr.getDouble("value_fund_ccy")));
tv.setAttribute("ccy", ccy);
el(doc, pos, "TotalPercentage",
f2(qr.getDouble("percentage")));
num(qr.getDouble("percentage"), 4, 2));
String kind = qr.getString("kind");
if (kind == null || !POSITION_KINDS.contains(kind))
kind = "Generic";
Element ke = el(doc, pos, kind, null);
Object q = qr.getObject("kind_qty");
if (QTY_ELEM.containsKey(kind) && q != null)
el(doc, ke, QTY_ELEM.get(kind),
f2(((Number) q).doubleValue()));
num(((Number) q).doubleValue(), 6, 2));
}
}

Expand Down Expand Up @@ -199,7 +216,7 @@ public static void main(String[] args) throws Exception {
el(doc, pe2, "PriceCurrency", sr.getString("currency"));
el(doc, pe2, "PriceNature", "OFFICIAL");
el(doc, pe2, "NavPrice",
f2(((Number) navp).doubleValue()));
num(((Number) navp).doubleValue(), 6, 2));
}
Object navf = sr.getObject("nav_fund_ccy");
if (navf != null) {
Expand All @@ -213,9 +230,8 @@ public static void main(String[] args) throws Exception {
a2.setAttribute("ccy", ccy);
Object so = sr.getObject("shares_outstanding");
if (so != null)
el(doc, t2, "SharesOutstanding", String.format(
Locale.ROOT, "%.0f",
((Number) so).doubleValue()));
el(doc, t2, "SharesOutstanding",
num(((Number) so).doubleValue(), 6, 0));
}
}
}
Expand Down
21 changes: 16 additions & 5 deletions Database_Integration/javascript/export_fundsxml.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,18 @@ const POSITION_KINDS = new Set(["Equity", "Bond", "ShareClass", "Warrant",
const QTY_ELEM = { Equity: "Units", Warrant: "Units", Certificate: "Units",
Bond: "Nominal", ShareClass: "Shares", Option: "Contracts",
Future: "Contracts" };
const f2 = (v) => Number(v).toFixed(2);
// Number formatting follows the DDL scale (schema.sql): amounts DECIMAL(20,2),
// TotalPercentage DECIMAL(9,4), quantities / NavPrice / SharesOutstanding
// DECIMAL(28,6). Render at that scale, then drop trailing zeros down to a floor
// of `minDec` decimals: 8.33 -> "8.33", 8.3333 -> "8.3333", 550000 shares ->
// "550000". A fixed toFixed(2) would silently truncate what the model stores
// (xml_equiv.py compares numerically and would flag the loss).
const num = (v, scale, minDec = 2) => {
const [whole, frac = ""] = Number(v).toFixed(scale).split(".");
const f = frac.replace(/0+$/, "").padEnd(minDec, "0");
return f ? `${whole}.${f}` : whole;
};
const f2 = (v) => num(v, 2); // amounts (scale 2)

function el(doc, parent, tag, text) { // the one XML-build primitive
const e = doc.createElement(tag);
Expand Down Expand Up @@ -119,11 +130,11 @@ for (const f of rows(db,
const tv = el(doc, el(doc, pos, "TotalValue"), "Amount",
f2(p.value_fund_ccy));
tv.setAttribute("ccy", ccy);
el(doc, pos, "TotalPercentage", f2(p.percentage));
el(doc, pos, "TotalPercentage", num(p.percentage, 4));
const kind = POSITION_KINDS.has(p.kind) ? p.kind : "Generic";
const ke = el(doc, pos, kind);
if (QTY_ELEM[kind] && p.kind_qty != null)
el(doc, ke, QTY_ELEM[kind], f2(p.kind_qty));
el(doc, ke, QTY_ELEM[kind], num(p.kind_qty, 6));
}
}

Expand All @@ -143,7 +154,7 @@ for (const f of rows(db,
el(doc, pr, "NavDate", f.nav_date);
el(doc, pr, "PriceCurrency", sc.currency);
el(doc, pr, "PriceNature", "OFFICIAL");
el(doc, pr, "NavPrice", f2(sc.nav_price));
el(doc, pr, "NavPrice", num(sc.nav_price, 6));
}
if (sc.nav_fund_ccy != null) {
const t2 = el(doc, el(doc, x, "TotalAssetValues"),
Expand All @@ -155,7 +166,7 @@ for (const f of rows(db,
a2.setAttribute("ccy", ccy);
if (sc.shares_outstanding != null)
el(doc, t2, "SharesOutstanding",
Number(sc.shares_outstanding).toFixed(0));
num(sc.shares_outstanding, 6, 0));
}
}
}
Expand Down
28 changes: 21 additions & 7 deletions Database_Integration/python/export_fundsxml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,20 @@
"Option": "Contracts", "Future": "Contracts"}


# Number formatting follows the DDL scale (schema.sql): amounts DECIMAL(20,2),
# TotalPercentage DECIMAL(9,4), quantities / NavPrice / SharesOutstanding
# DECIMAL(28,6). Render with that scale, then drop trailing zeros down to a
# floor of `min_dec` decimals, so 8.33 -> "8.33", 8.3333 -> "8.3333",
# 50000.123456 -> "50000.123456" and 550000 shares -> "550000". A fixed "%.2f"
# would silently truncate anything the model can store with more decimals
# (xml_equiv.py compares numerically, so it would flag the loss).
def _num(v, scale, min_dec=2):
s = f"{float(v):.{scale}f}"
whole, _, frac = s.partition(".")
frac = frac.rstrip("0")
frac = frac.ljust(min_dec, "0")
return f"{whole}.{frac}" if frac else whole

def _el(parent, tag, text=None, **attrs):
"""Append a child element — the single XML-building primitive."""
e = etree.SubElement(parent, tag)
Expand Down Expand Up @@ -106,7 +120,7 @@ def main() -> int:
"NavDate", f["nav_date"]).getparent()
_el(tav, "TotalAssetNature", "OFFICIAL")
_el(_el(tav, "TotalNetAssetValue"), "Amount",
f'{f["total_nav"]:.2f}', ccy=ccy)
_num(f["total_nav"], 2), ccy=ccy)

ports = _el(fdd, "Portfolios")
for pf in con.execute(
Expand All @@ -126,12 +140,12 @@ def main() -> int:
if p["currency"]:
_el(pos, "Currency", p["currency"])
_el(_el(pos, "TotalValue"), "Amount",
f'{p["value_fund_ccy"]:.2f}', ccy=ccy)
_el(pos, "TotalPercentage", f'{p["percentage"]:.2f}')
_num(p["value_fund_ccy"], 2), ccy=ccy)
_el(pos, "TotalPercentage", _num(p["percentage"], 4))
kind = p["kind"] if p["kind"] in POSITION_KINDS else "Generic"
ke = _el(pos, kind)
if kind in QTY_ELEM and p["kind_qty"] is not None:
_el(ke, QTY_ELEM[kind], f'{p["kind_qty"]:.2f}')
_el(ke, QTY_ELEM[kind], _num(p["kind_qty"], 6))

scs = con.execute(
"SELECT * FROM share_class WHERE document_id=? AND fund_seq=? "
Expand All @@ -150,16 +164,16 @@ def main() -> int:
_el(pr, "NavDate", f["nav_date"])
_el(pr, "PriceCurrency", sc["currency"])
_el(pr, "PriceNature", "OFFICIAL")
_el(pr, "NavPrice", f'{sc["nav_price"]:.2f}')
_el(pr, "NavPrice", _num(sc["nav_price"], 6))
if sc["nav_fund_ccy"] is not None:
t = _el(_el(x, "TotalAssetValues"), "TotalAssetValue")
_el(t, "NavDate", f["nav_date"])
_el(t, "TotalAssetNature", "OFFICIAL")
_el(_el(t, "TotalNetAssetValue"), "Amount",
f'{sc["nav_fund_ccy"]:.2f}', ccy=ccy)
_num(sc["nav_fund_ccy"], 2), ccy=ccy)
if sc["shares_outstanding"] is not None:
_el(t, "SharesOutstanding",
f'{sc["shares_outstanding"]:.0f}')
_num(sc["shares_outstanding"], 6, 0))

assets = con.execute(
"SELECT * FROM asset WHERE document_id=? ORDER BY unique_id",
Expand Down