Entity Framework not updating one-to-one entities

Question

I have these models:

public class Address
{
    public int Id {get; set;}
    public string Street {get; set;}
    public string HouseNo {get; set;}
}

public class Customer
{
    public int Id {get; set;}
    public string Name {get; set;}
    public int AddressId {get; set;}
    public virtual Address Address {get; set;}
}

And the mapping classes:

public class AddressMap : EntityTypeConfiguration<Address>
{
    public AddressMap()
    {
        this.ToTable("addresses");
        this.HasKey(t => t.Id).Property(t => t.Id).HasColumnName("id")
           .HasDatabaseGeneratedOption(DatabaseGeneratedOption.Identity);
        this.Property(t => t.Street).HasColumnName("street");
        this.Property(t => t.HouseNumber).HasColumnName("house_no");
     }
}

public class CustomerMap : EntityTypeConfiguration<Customer>
{
    public CustomerMap()
    {
        this.ToTable("customers");
        this.HasKey(t => t.Id).Property(t => t.Id).HasColumnName("id")
           .HasDatabaseGeneratedOption(DatabaseGeneratedOption.Identity);

        this.Property(t => t.Name).HasColumnName("name");
        this.Property(t => t.AddressId).HasColumnName("id_address");

        this.HasOptional(t => t.Address)
            .WithMany()
            .HasForeignKey(t => t.AddressId);
    }
}

I'm using GraphDiff for CRUD operations.

Adding a new customer with an address works fine. However, when I want to remove a customer's address by doing the following:

if (customer.Address != null)
{
    if (customer.Address.IsEmpty())
    {
        if (customer.Address.Id > 0)
            addressRepository.Delete(customer.Address);

            customer.Address = null;
    }
}

customerRepository.Update(customer);

And in the repository:

public override Customer Update(Customer entity)
{
    return ((DbContext)dataContext).UpdateGraph<Customer>
            (entity,
                map => map.OwnedEntity(x => x.Address)
            );      
}

The customer's address is set to NULL in the db. However, the address is not being deleted from the db.

Is it a wrong configuration in my mapping classes or is GraphDiff not behaving as expected?