Collider[] results =
    Physics.OverlapSphere(
        transform.position,
        radius,
        targetLayer
    );

Transform nearest = null;
float bestDistance =
    Mathf.Infinity;

foreach (Collider col in results)
{
    float distance =
        Vector3.SqrMagnitude(
            col.transform.position -
            transform.position
        );

    if (distance < bestDistance)
    {
        bestDistance = distance;
        nearest = col.transform;
    }
}