possible bug related to history for isComponent

[Kevin Ludwig]

I've got a data model for movies (since I was following the tutorial on
datomic.com!) and I decided to add a little bit to it. In particular I put
an :isComponent on the movie for rating that looks like this:

{:db/id #db/id[:db.part/db]

:db/ident :movie/title

:db/valueType :db.type/string

:db/cardinality :db.cardinality/one

:db/doc "the title of the movie"}



{:db/id #db/id[:db.part/db]

:db/ident :movie/release-year

:db/valueType :db.type/long

:db/cardinality :db.cardinality/one

:db/doc "original theatrical release year"}


{:db/id #db/id[:db.part/db]

:db/ident :rating/value

:db/valueType :db.type/string

:db/cardinality :db.cardinality/one

:db/doc "one of NR, G, PG, PG13, R, NC17"}



{:db/id #db/id[:db.part/db]

:db/ident :rating/source

:db/valueType :db.type/string

:db/cardinality :db.cardinality/one

:db/doc "the rating body, e.g. MPAA"}


{:db/id #db/id[:db.part/db]

:db/ident :rating/advisories

:db/valueType :db.type/string

:db/cardinality :db.cardinality/many

:db/doc "rating advisories, e.g. adult language, ..."}


; I use these helpers throughout:

(declare conn)


(defn get-conn []

(let [uri (config/get-key :db :uri)]

(defonce conn (d/connect uri))

conn))



(defn get-db []

(let [conn (get-conn)]

(d/db conn)))


; Then I added a document like this:

(defn create []

(let [cxn (get-conn)

datom {:db/id (d/tempid :db.part/user) :movie/title "Some Movie"
:movie/release-year 2017 :movie/rating {:rating/value "R"}}

tx @(d/transact cxn [datom tx-datom])]

(d/resolve-tempid (get-db) (:tempids tx) (:db/id datom))))


; Then I updated the rating and title like this:

(defn update [id]

(let [cxn (get-conn)

datom {:db/id id :title "Some Movie 2" :movie/rating
{:rating/value "PG"}}

tx @(d/transact cxn [datom tx-datom])]

(d/pull (get-db) '[*] id)))


; Then I pulled history on :movie/title:

(defn movie-title-history [id]

(let [hdb (d/history (get-db))

attr :movie/title

query '[:find ?e ?a ?v ?t ?op

:in $ ?e ?a

:where [?e ?a ?v ?t ?op]]]

(d/q query hdb id attr)))


; and then on :rating/value

(defn movie-rating-value-history [id]

(let [hdb (d/history (get-db))

attr :rating/value

query '[:find ?e ?a ?v ?t ?op

:in $ ?e ?a

:where [?e _ ?ref]

[?ref ?a ?v ?t ?op]]]

(d/q query hdb id attr)))


And I was surprised to find that my history on title had 3 entries (2 adds
and 1 retract), whereas my history on :rating/value has only 2 entries (2
adds, and no retract). Is this a bug or am I doing something wrong? For
reference I'm using version datomic-pro-0.9.5561.

[Kevin Ludwig]

Sorry...I managed to miss one schema element above, :movie/rating

{:db/id #db/id[:db.part/db]

:db/ident :movie/rating

:db/valueType :db.type/ref

:db/isComponent true

:db/cardinality :db.cardinality/one

:db/doc "rating information"}

Post by Kevin Ludwig
I've got a data model for movies (since I was following the tutorial on
datomic.com!) and I decided to add a little bit to it. In particular I
{:db/id #db/id[:db.part/db]
:db/ident :movie/title
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the title of the movie"}
{:db/id #db/id[:db.part/db]
:db/ident :movie/release-year
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one
:db/doc "original theatrical release year"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/value
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "one of NR, G, PG, PG13, R, NC17"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/source
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the rating body, e.g. MPAA"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/advisories
:db/valueType :db.type/string
:db/cardinality :db.cardinality/many
:db/doc "rating advisories, e.g. adult language, ..."}
(declare conn)
(defn get-conn []
(let [uri (config/get-key :db :uri)]
(defonce conn (d/connect uri))
conn))
(defn get-db []
(let [conn (get-conn)]
(d/db conn)))
(defn create []
(let [cxn (get-conn)
datom {:db/id (d/tempid :db.part/user) :movie/title "Some
Movie" :movie/release-year 2017 :movie/rating {:rating/value "R"}}
(d/resolve-tempid (get-db) (:tempids tx) (:db/id datom))))
(defn update [id]
(let [cxn (get-conn)
datom {:db/id id :title "Some Movie 2" :movie/rating
{:rating/value "PG"}}
(d/pull (get-db) '[*] id)))
(defn movie-title-history [id]
(let [hdb (d/history (get-db))
attr :movie/title
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
; and then on :rating/value
(defn movie-rating-value-history [id]
(let [hdb (d/history (get-db))
attr :rating/value
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e _ ?ref]
[?ref ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
And I was surprised to find that my history on title had 3 entries (2 adds
and 1 retract), whereas my history on :rating/value has only 2 entries (2
adds, and no retract). Is this a bug or am I doing something wrong? For
reference I'm using version datomic-pro-0.9.5561.

[Francis Avila]

When you transact like this:

{:db/id id :title "Some Movie 2" :movie/rating {:rating/value "PG"}}

the old :movie/rating is retracted (because it is cardinality-one) and a
new entity with {:rating/value "PG"} is asserted. Thus it has no history,
since this is a new entity.

Were you perhaps expecting that the transaction would choose the existing
:db/id for :movie/rating as the :db/id for {:rating/value "PG"}? Datomic
never does this, isComponent or not. Entity id resolution when :db/id is
not supplied either finds a match via a :db/identity attribute, or it mints
a new one in the partition of the parent map (isComponent attribute) or in
the user partition (other cases).

Post by Kevin Ludwig
I've got a data model for movies (since I was following the tutorial on
datomic.com!) and I decided to add a little bit to it. In particular I
{:db/id #db/id[:db.part/db]
:db/ident :movie/title
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the title of the movie"}
{:db/id #db/id[:db.part/db]
:db/ident :movie/release-year
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one
:db/doc "original theatrical release year"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/value
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "one of NR, G, PG, PG13, R, NC17"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/source
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the rating body, e.g. MPAA"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/advisories
:db/valueType :db.type/string
:db/cardinality :db.cardinality/many
:db/doc "rating advisories, e.g. adult language, ..."}
(declare conn)
(defn get-conn []
(let [uri (config/get-key :db :uri)]
(defonce conn (d/connect uri))
conn))
(defn get-db []
(let [conn (get-conn)]
(d/db conn)))
(defn create []
(let [cxn (get-conn)
datom {:db/id (d/tempid :db.part/user) :movie/title "Some
Movie" :movie/release-year 2017 :movie/rating {:rating/value "R"}}
(d/resolve-tempid (get-db) (:tempids tx) (:db/id datom))))
(defn update [id]
(let [cxn (get-conn)
datom {:db/id id :title "Some Movie 2" :movie/rating
{:rating/value "PG"}}
(d/pull (get-db) '[*] id)))
(defn movie-title-history [id]
(let [hdb (d/history (get-db))
attr :movie/title
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
; and then on :rating/value
(defn movie-rating-value-history [id]
(let [hdb (d/history (get-db))
attr :rating/value
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e _ ?ref]
[?ref ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
And I was surprised to find that my history on title had 3 entries (2 adds
and 1 retract), whereas my history on :rating/value has only 2 entries (2
adds, and no retract). Is this a bug or am I doing something wrong? For
reference I'm using version datomic-pro-0.9.5561.

[Kevin Ludwig]

Hi, Thanks.

I guess what I was expecting was that the retraction would be on the old
:rating/value, although what you say makes sense. I guess that means that
if I had a :rating/source or :rating/advisories set on the prior
:movie/rating object, I would have lost those? And if I really do want to
create history on the :rating/value (and not on :movie/rating) I should
have transacted with the ratingId, like below?

{:db/id id :title "Some Movie 2" :movie/rating {:db/id ratingId
:rating/value "PG"}}

Post by Francis Avila
{:db/id id :title "Some Movie 2" :movie/rating {:rating/value "PG"}}
the old :movie/rating is retracted (because it is cardinality-one) and a
new entity with {:rating/value "PG"} is asserted. Thus it has no history,
since this is a new entity.
Were you perhaps expecting that the transaction would choose the existing
:db/id for :movie/rating as the :db/id for {:rating/value "PG"}? Datomic
never does this, isComponent or not. Entity id resolution when :db/id is
not supplied either finds a match via a :db/identity attribute, or it mints
a new one in the partition of the parent map (isComponent attribute) or in
the user partition (other cases).

Post by Kevin Ludwig
I've got a data model for movies (since I was following the tutorial on
datomic.com!) and I decided to add a little bit to it. In particular I
{:db/id #db/id[:db.part/db]
:db/ident :movie/title
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the title of the movie"}
{:db/id #db/id[:db.part/db]
:db/ident :movie/release-year
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one
:db/doc "original theatrical release year"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/value
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "one of NR, G, PG, PG13, R, NC17"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/source
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the rating body, e.g. MPAA"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/advisories
:db/valueType :db.type/string
:db/cardinality :db.cardinality/many
:db/doc "rating advisories, e.g. adult language, ..."}
(declare conn)
(defn get-conn []
(let [uri (config/get-key :db :uri)]
(defonce conn (d/connect uri))
conn))
(defn get-db []
(let [conn (get-conn)]
(d/db conn)))
(defn create []
(let [cxn (get-conn)
datom {:db/id (d/tempid :db.part/user) :movie/title "Some
Movie" :movie/release-year 2017 :movie/rating {:rating/value "R"}}
(d/resolve-tempid (get-db) (:tempids tx) (:db/id datom))))
(defn update [id]
(let [cxn (get-conn)
datom {:db/id id :title "Some Movie 2" :movie/rating
{:rating/value "PG"}}
(d/pull (get-db) '[*] id)))
(defn movie-title-history [id]
(let [hdb (d/history (get-db))
attr :movie/title
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
; and then on :rating/value
(defn movie-rating-value-history [id]
(let [hdb (d/history (get-db))
attr :rating/value
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e _ ?ref]
[?ref ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
And I was surprised to find that my history on title had 3 entries (2
adds and 1 retract), whereas my history on :rating/value has only 2 entries
(2 adds, and no retract). Is this a bug or am I doing something wrong? For
reference I'm using version datomic-pro-0.9.5561.

[Francis Avila]

Post by Kevin Ludwig
Hi, Thanks.
I guess what I was expecting was that the retraction would be on the old
:rating/value, although what you say makes sense.

Remember that the map form of transactions is purely syntactic sugar for a
bunch of :db/add assertions on the same entity-id. It is not a magic
graph-merging syntax because no one merge algorithm can please everyone or
satisfy all use cases. The rule for determining the id of something missing
is simple and the map expansion to :db/add assertions is nearly mechanical.

I guess that means that if I had a :rating/source or :rating/advisories set

Post by Kevin Ludwig
on the prior :movie/rating object, I would have lost those?

Yes.

Post by Kevin Ludwig
And if I really do want to create history on the :rating/value (and not on
:movie/rating) I should have transacted with the ratingId, like below?
{:db/id id :title "Some Movie 2" :movie/rating {:db/id ratingId
:rating/value "PG"}}

You could do this. You could also do two separate assertions: [{:db/id id
:title "Some Movie 2" } {:db/id ratingId :rating/value "PG"}]

The difference lies in what happens if there is a race condition. Suppose
some other peer changed :movie/rating to point at a new component id and
its transaction applied first, but you submitted a new transaction before
you could see the result.

E.g. other peer: [{:db/id id :movie/rating {:rating/value "G"}}]


You don't know the new ratingId, and the old rating is now orphaned (i.e.
still has assertions on it, but no one references it).


Your first transaction (the nested map) will re-assert the old rating
entity and also update its :rating/value.


The second one (mine, with two maps) will update :rating/value on the
orphaned rating entity, but not reassert the :movie/value connection.


Neither behavior is intrinsically right or wrong: it depends on what you
want in a particular situation.


If the other peer had instead retracted the rating completely with
[[:db.fn/retractEntity ratingId]] (maybe to keep from making orphans), then
your transactions will be updating an empty rating entity.


So you need to at least think about situations like this when using refs.
If absolute atomicity is required, write a transaction function so it is
performed atomically on the transactor. Alternatively, you can use
:db.fn/cas http://docs.datomic.com/transactions.html#sec-4-2 to assert that
nothing has changed, or you can write transaction functions which do the
same thing but e.g., assert that an entity-plus-components hasn't been
changed.

Post by Kevin Ludwig

Post by Francis Avila
{:db/id id :title "Some Movie 2" :movie/rating {:rating/value "PG"}}
the old :movie/rating is retracted (because it is cardinality-one) and a
new entity with {:rating/value "PG"} is asserted. Thus it has no history,
since this is a new entity.
Were you perhaps expecting that the transaction would choose the existing
:db/id for :movie/rating as the :db/id for {:rating/value "PG"}? Datomic
never does this, isComponent or not. Entity id resolution when :db/id is
not supplied either finds a match via a :db/identity attribute, or it mints
a new one in the partition of the parent map (isComponent attribute) or in
the user partition (other cases).

Post by Kevin Ludwig
I've got a data model for movies (since I was following the tutorial on
datomic.com!) and I decided to add a little bit to it. In particular I
{:db/id #db/id[:db.part/db]
:db/ident :movie/title
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the title of the movie"}
{:db/id #db/id[:db.part/db]
:db/ident :movie/release-year
:db/valueType :db.type/long
:db/cardinality :db.cardinality/one
:db/doc "original theatrical release year"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/value
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "one of NR, G, PG, PG13, R, NC17"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/source
:db/valueType :db.type/string
:db/cardinality :db.cardinality/one
:db/doc "the rating body, e.g. MPAA"}
{:db/id #db/id[:db.part/db]
:db/ident :rating/advisories
:db/valueType :db.type/string
:db/cardinality :db.cardinality/many
:db/doc "rating advisories, e.g. adult language, ..."}
(declare conn)
(defn get-conn []
(let [uri (config/get-key :db :uri)]
(defonce conn (d/connect uri))
conn))
(defn get-db []
(let [conn (get-conn)]
(d/db conn)))
(defn create []
(let [cxn (get-conn)
datom {:db/id (d/tempid :db.part/user) :movie/title "Some
Movie" :movie/release-year 2017 :movie/rating {:rating/value "R"}}
(d/resolve-tempid (get-db) (:tempids tx) (:db/id datom))))
(defn update [id]
(let [cxn (get-conn)
datom {:db/id id :title "Some Movie 2" :movie/rating
{:rating/value "PG"}}
(d/pull (get-db) '[*] id)))
(defn movie-title-history [id]
(let [hdb (d/history (get-db))
attr :movie/title
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
; and then on :rating/value
(defn movie-rating-value-history [id]
(let [hdb (d/history (get-db))
attr :rating/value
query '[:find ?e ?a ?v ?t ?op
:in $ ?e ?a
:where [?e _ ?ref]
[?ref ?a ?v ?t ?op]]]
(d/q query hdb id attr)))
And I was surprised to find that my history on title had 3 entries (2
adds and 1 retract), whereas my history on :rating/value has only 2 entries
(2 adds, and no retract). Is this a bug or am I doing something wrong? For
reference I'm using version datomic-pro-0.9.5561.