Зрители на стриме помогли найти целых 2 решения проблемы:
1) Многословное

abstract class Factory<TProduct, TItem, TFactory>
    where TProduct : Product<TProduct, TItem, TFactory>
    where TItem : Object
    where TFactory : Factory<TProduct, TItem, TFactory>
{
    public int counter;
    public abstract TProduct Make();
}
class Product<TProduct, TItem, TFactory>
    where TProduct : Product<TProduct, TItem, TFactory>
    where TItem : Object
    where TFactory : Factory<TProduct, TItem, TFactory>
{
    private readonly TFactory factory;
    protected Product(TFactory factory)
    {
        this.factory = factory;
        factory.counter++;
    }
    public void Destroy()
    {
        factory.counter--;
    }
}
class FloatFactory : Factory<FloatProduct, Object, FloatFactory>
{
    public override FloatProduct Make()
    {
        return new FloatProduct(this);
    }
}
class FloatProduct : Product<FloatProduct, Object, FloatFactory>
{
    public FloatProduct(FloatFactory factory) : base(factory)
    {
    }
}

2) с ковариантом

interface IFactory<out TProduct>
{
    int Counter { get; set; }
    TProduct Make();
}

class Product<TItem>
{
    private readonly IFactory<Product<TItem>> factory;

    protected Product(IFactory<Product<TItem>> factory)
    {
        this.factory = factory;
        factory.Counter++;
    }

    public void Destroy()
    {
        factory.Counter--;
    }
}

class FloatFactory : IFactory<FloatProduct>
{
    public int Counter { get; set; }

    public FloatProduct Make()
    {
        return new FloatProduct(this);
    }
}

class FloatProduct : Product<float>
{
    public FloatProduct(FloatFactory factory) : base(factory)
    {
    }
}