Confusing behaviour on form submit with session_state

I wish to update a variable stored in session_state on submitting a form, however on hitting Submit I see the button change to true but the variable is not updated. I have to hit Submit a second time to update the variable. What is going on here? Code at bottom

import streamlit as st
import glob as glob

file_pattern = st.text_input('Glob file pattern to match', 'images/*.png')
image_list = glob.glob(file_pattern)
st.write(f"Found {len(image_list)} images")

def refresh():
    st.session_state["image_index"] = 0
    st.session_state["annotations"] = []

if "image_index" not in st.session_state:
    refresh()

st.sidebar.button('Refresh', on_click=refresh)
st.sidebar.write(image_list)
st.sidebar.write(st.session_state)

image_index = st.session_state["image_index"]
st.image(image_list[image_index], caption = f"{image_list[image_index]}", use_column_width=True)

with st.form(f"annotation_form_{image_index}"):
    choice = st.radio("Choose", ("keep", "discard"))
    if st.form_submit_button("Submit"):
        st.session_state["annotations"].append({"image": image_list[image_index], "choice": choice})
        st.session_state["image_index"] += 1
        

if len(st.session_state["annotations"]) > 0:
    st.dataframe(data=st.session_state["annotations"])